2024-11-13 14:13:32,769 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-13 14:13:32,786 main DEBUG Took 0.015323 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 14:13:32,786 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 14:13:32,787 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 14:13:32,788 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 14:13:32,789 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,795 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 14:13:32,807 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,808 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,809 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,809 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,810 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,810 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,811 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,811 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,812 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,812 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,813 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,813 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,813 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,814 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,814 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,814 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,815 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,815 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,816 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,816 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,816 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,816 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,817 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,817 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 14:13:32,818 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,818 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 14:13:32,819 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 14:13:32,821 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 14:13:32,822 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 14:13:32,823 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 14:13:32,824 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 14:13:32,824 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 14:13:32,832 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 14:13:32,835 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 14:13:32,836 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 14:13:32,837 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 14:13:32,837 main DEBUG createAppenders(={Console}) 2024-11-13 14:13:32,838 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-13 14:13:32,838 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-13 14:13:32,838 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-13 14:13:32,839 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 14:13:32,839 main DEBUG OutputStream closed 2024-11-13 14:13:32,839 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 14:13:32,839 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 14:13:32,840 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-13 14:13:32,913 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 14:13:32,915 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 14:13:32,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 14:13:32,916 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 14:13:32,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 14:13:32,917 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 14:13:32,918 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 14:13:32,918 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 14:13:32,918 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 14:13:32,918 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 14:13:32,919 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 14:13:32,919 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 14:13:32,919 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 14:13:32,920 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 14:13:32,920 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 14:13:32,920 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 14:13:32,920 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 14:13:32,921 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 14:13:32,923 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 14:13:32,923 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-13 14:13:32,924 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 14:13:32,924 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-13T14:13:32,935 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-13 14:13:32,938 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 14:13:32,938 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T14:13:33,231 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036 2024-11-13T14:13:33,250 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T14:13:33,266 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b, deleteOnExit=true 2024-11-13T14:13:33,266 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T14:13:33,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/test.cache.data in system properties and HBase conf 2024-11-13T14:13:33,268 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T14:13:33,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/hadoop.log.dir in system properties and HBase conf 2024-11-13T14:13:33,269 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T14:13:33,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T14:13:33,270 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T14:13:33,354 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T14:13:33,435 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T14:13:33,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T14:13:33,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T14:13:33,440 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T14:13:33,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T14:13:33,441 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T14:13:33,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T14:13:33,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T14:13:33,442 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T14:13:33,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T14:13:33,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/nfs.dump.dir in system properties and HBase conf 2024-11-13T14:13:33,443 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/java.io.tmpdir in system properties and HBase conf 2024-11-13T14:13:33,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T14:13:33,444 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T14:13:33,445 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T14:13:34,497 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T14:13:34,568 INFO [Time-limited test {}] log.Log(170): Logging initialized @2451ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T14:13:34,636 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T14:13:34,697 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T14:13:34,722 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T14:13:34,722 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T14:13:34,723 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T14:13:34,735 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T14:13:34,738 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/hadoop.log.dir/,AVAILABLE} 2024-11-13T14:13:34,739 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T14:13:34,907 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@376fa8f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/java.io.tmpdir/jetty-localhost-38501-hadoop-hdfs-3_4_1-tests_jar-_-any-17258420583358725956/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T14:13:34,917 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:38501} 2024-11-13T14:13:34,918 INFO [Time-limited test {}] server.Server(415): Started @2801ms 2024-11-13T14:13:35,490 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T14:13:35,499 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T14:13:35,500 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T14:13:35,500 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T14:13:35,500 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T14:13:35,501 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@40b6b75c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/hadoop.log.dir/,AVAILABLE} 2024-11-13T14:13:35,502 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@70201ae6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T14:13:35,605 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@715133de{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/java.io.tmpdir/jetty-localhost-37067-hadoop-hdfs-3_4_1-tests_jar-_-any-10406342909795564235/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T14:13:35,606 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6d8e9708{HTTP/1.1, (http/1.1)}{localhost:37067} 2024-11-13T14:13:35,606 INFO [Time-limited test {}] server.Server(415): Started @3490ms 2024-11-13T14:13:35,654 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T14:13:35,753 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T14:13:35,760 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T14:13:35,762 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T14:13:35,763 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T14:13:35,763 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T14:13:35,764 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@9c3cb41{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/hadoop.log.dir/,AVAILABLE} 2024-11-13T14:13:35,765 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2041eb0d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T14:13:35,901 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1873c69b{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/java.io.tmpdir/jetty-localhost-38089-hadoop-hdfs-3_4_1-tests_jar-_-any-5062344220849199499/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T14:13:35,902 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6ca981a0{HTTP/1.1, (http/1.1)}{localhost:38089} 2024-11-13T14:13:35,902 INFO [Time-limited test {}] server.Server(415): Started @3786ms 2024-11-13T14:13:35,905 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T14:13:35,945 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T14:13:35,950 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T14:13:35,952 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T14:13:35,953 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T14:13:35,953 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T14:13:35,954 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48261703{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/hadoop.log.dir/,AVAILABLE} 2024-11-13T14:13:35,954 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@796aba77{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T14:13:36,060 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@27d2afd6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/java.io.tmpdir/jetty-localhost-39863-hadoop-hdfs-3_4_1-tests_jar-_-any-8922437219734139056/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T14:13:36,061 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1f351b31{HTTP/1.1, (http/1.1)}{localhost:39863} 2024-11-13T14:13:36,061 INFO [Time-limited test {}] server.Server(415): Started @3944ms 2024-11-13T14:13:36,063 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T14:13:37,494 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data4/current/BP-1546972301-172.17.0.2-1731507214000/current, will proceed with Du for space computation calculation, 2024-11-13T14:13:37,494 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data3/current/BP-1546972301-172.17.0.2-1731507214000/current, will proceed with Du for space computation calculation, 2024-11-13T14:13:37,494 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data2/current/BP-1546972301-172.17.0.2-1731507214000/current, will proceed with Du for space computation calculation, 2024-11-13T14:13:37,494 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data1/current/BP-1546972301-172.17.0.2-1731507214000/current, will proceed with Du for space computation calculation, 2024-11-13T14:13:37,524 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T14:13:37,524 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T14:13:37,531 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data5/current/BP-1546972301-172.17.0.2-1731507214000/current, will proceed with Du for space computation calculation, 2024-11-13T14:13:37,532 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data6/current/BP-1546972301-172.17.0.2-1731507214000/current, will proceed with Du for space computation calculation, 2024-11-13T14:13:37,555 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T14:13:37,576 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1bb425dc0dbbae5f with lease ID 0xccc8c86b9186beb9: Processing first storage report for DS-0b2f9f3b-9824-4059-870f-9bb615a80b01 from datanode DatanodeRegistration(127.0.0.1:46747, datanodeUuid=c9bb1ff7-beb3-4bf2-b8ea-918fdcc719f3, infoPort=42835, infoSecurePort=0, ipcPort=38281, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000) 2024-11-13T14:13:37,577 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1bb425dc0dbbae5f with lease ID 0xccc8c86b9186beb9: from storage DS-0b2f9f3b-9824-4059-870f-9bb615a80b01 node DatanodeRegistration(127.0.0.1:46747, datanodeUuid=c9bb1ff7-beb3-4bf2-b8ea-918fdcc719f3, infoPort=42835, infoSecurePort=0, ipcPort=38281, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T14:13:37,578 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5a283545af326935 with lease ID 0xccc8c86b9186beb8: Processing first storage report for DS-9b4f3d38-26ab-4b10-b6b8-307e24d8051e from datanode DatanodeRegistration(127.0.0.1:33015, datanodeUuid=1726ac66-1101-4851-b1bf-95c200689e57, infoPort=39941, infoSecurePort=0, ipcPort=45601, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000) 2024-11-13T14:13:37,578 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5a283545af326935 with lease ID 0xccc8c86b9186beb8: from storage DS-9b4f3d38-26ab-4b10-b6b8-307e24d8051e node DatanodeRegistration(127.0.0.1:33015, datanodeUuid=1726ac66-1101-4851-b1bf-95c200689e57, infoPort=39941, infoSecurePort=0, ipcPort=45601, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T14:13:37,578 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x61fd7ce27f03263 with lease ID 0xccc8c86b9186beba: Processing first storage report for DS-7d222e8a-09d0-41ad-8291-a32be5623236 from datanode DatanodeRegistration(127.0.0.1:35125, datanodeUuid=253dc82c-cfad-4b53-804e-f665b53be144, infoPort=43485, infoSecurePort=0, ipcPort=46493, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000) 2024-11-13T14:13:37,578 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x61fd7ce27f03263 with lease ID 0xccc8c86b9186beba: from storage DS-7d222e8a-09d0-41ad-8291-a32be5623236 node DatanodeRegistration(127.0.0.1:35125, datanodeUuid=253dc82c-cfad-4b53-804e-f665b53be144, infoPort=43485, infoSecurePort=0, ipcPort=46493, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T14:13:37,578 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1bb425dc0dbbae5f with lease ID 0xccc8c86b9186beb9: Processing first storage report for DS-c2c382e4-60a3-4e0c-b11e-1b2a5d2ff7e9 from datanode DatanodeRegistration(127.0.0.1:46747, datanodeUuid=c9bb1ff7-beb3-4bf2-b8ea-918fdcc719f3, infoPort=42835, infoSecurePort=0, ipcPort=38281, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000) 2024-11-13T14:13:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1bb425dc0dbbae5f with lease ID 0xccc8c86b9186beb9: from storage DS-c2c382e4-60a3-4e0c-b11e-1b2a5d2ff7e9 node DatanodeRegistration(127.0.0.1:46747, datanodeUuid=c9bb1ff7-beb3-4bf2-b8ea-918fdcc719f3, infoPort=42835, infoSecurePort=0, ipcPort=38281, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T14:13:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x5a283545af326935 with lease ID 0xccc8c86b9186beb8: Processing first storage report for DS-f155cab6-874d-44f2-b5a4-16218fb850fe from datanode DatanodeRegistration(127.0.0.1:33015, datanodeUuid=1726ac66-1101-4851-b1bf-95c200689e57, infoPort=39941, infoSecurePort=0, ipcPort=45601, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000) 2024-11-13T14:13:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x5a283545af326935 with lease ID 0xccc8c86b9186beb8: from storage DS-f155cab6-874d-44f2-b5a4-16218fb850fe node DatanodeRegistration(127.0.0.1:33015, datanodeUuid=1726ac66-1101-4851-b1bf-95c200689e57, infoPort=39941, infoSecurePort=0, ipcPort=45601, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T14:13:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x61fd7ce27f03263 with lease ID 0xccc8c86b9186beba: Processing first storage report for DS-d76b5666-3c0e-412e-af5f-9a879ff7db18 from datanode DatanodeRegistration(127.0.0.1:35125, datanodeUuid=253dc82c-cfad-4b53-804e-f665b53be144, infoPort=43485, infoSecurePort=0, ipcPort=46493, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000) 2024-11-13T14:13:37,579 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x61fd7ce27f03263 with lease ID 0xccc8c86b9186beba: from storage DS-d76b5666-3c0e-412e-af5f-9a879ff7db18 node DatanodeRegistration(127.0.0.1:35125, datanodeUuid=253dc82c-cfad-4b53-804e-f665b53be144, infoPort=43485, infoSecurePort=0, ipcPort=46493, storageInfo=lv=-57;cid=testClusterID;nsid=325364883;c=1731507214000), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T14:13:37,594 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036 2024-11-13T14:13:37,652 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/zookeeper_0, clientPort=59966, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T14:13:37,661 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59966 2024-11-13T14:13:37,670 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T14:13:37,673 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T14:13:37,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741825_1001 (size=7) 2024-11-13T14:13:37,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741825_1001 (size=7) 2024-11-13T14:13:37,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741825_1001 (size=7) 2024-11-13T14:13:38,283 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b with version=8 2024-11-13T14:13:38,284 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/hbase-staging 2024-11-13T14:13:38,359 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T14:13:38,641 INFO [Time-limited test {}] client.ConnectionUtils(128): master/eb1966fd6578:0 server-side Connection retries=45 2024-11-13T14:13:38,649 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:38,650 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:38,654 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T14:13:38,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:38,654 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T14:13:38,789 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T14:13:38,845 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T14:13:38,854 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T14:13:38,858 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T14:13:38,883 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3754 (auto-detected) 2024-11-13T14:13:38,884 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-13T14:13:38,903 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36813 2024-11-13T14:13:38,923 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36813 connecting to ZooKeeper ensemble=127.0.0.1:59966 2024-11-13T14:13:39,129 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:368130x0, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T14:13:39,137 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36813-0x101348b60910000 connected 2024-11-13T14:13:39,240 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T14:13:39,243 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T14:13:39,253 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T14:13:39,257 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b, hbase.cluster.distributed=false 2024-11-13T14:13:39,282 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T14:13:39,288 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36813 2024-11-13T14:13:39,289 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36813 2024-11-13T14:13:39,290 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36813 2024-11-13T14:13:39,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36813 2024-11-13T14:13:39,296 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36813 2024-11-13T14:13:39,406 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eb1966fd6578:0 server-side Connection retries=45 2024-11-13T14:13:39,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:39,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:39,408 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T14:13:39,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:39,408 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T14:13:39,411 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T14:13:39,414 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T14:13:39,415 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38145 2024-11-13T14:13:39,418 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38145 connecting to ZooKeeper ensemble=127.0.0.1:59966 2024-11-13T14:13:39,419 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T14:13:39,424 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T14:13:39,438 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:381450x0, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T14:13:39,439 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:381450x0, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T14:13:39,439 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38145-0x101348b60910001 connected 2024-11-13T14:13:39,442 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T14:13:39,449 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T14:13:39,452 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T14:13:39,457 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T14:13:39,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38145 2024-11-13T14:13:39,458 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38145 2024-11-13T14:13:39,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38145 2024-11-13T14:13:39,459 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38145 2024-11-13T14:13:39,460 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38145 2024-11-13T14:13:39,475 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eb1966fd6578:0 server-side Connection retries=45 2024-11-13T14:13:39,475 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:39,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:39,476 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T14:13:39,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:39,476 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T14:13:39,477 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T14:13:39,477 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T14:13:39,478 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:42135 2024-11-13T14:13:39,479 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42135 connecting to ZooKeeper ensemble=127.0.0.1:59966 2024-11-13T14:13:39,480 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T14:13:39,484 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T14:13:39,501 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:421350x0, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T14:13:39,502 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:421350x0, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T14:13:39,502 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42135-0x101348b60910002 connected 2024-11-13T14:13:39,502 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T14:13:39,503 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T14:13:39,504 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T14:13:39,506 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T14:13:39,507 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42135 2024-11-13T14:13:39,508 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42135 2024-11-13T14:13:39,509 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42135 2024-11-13T14:13:39,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42135 2024-11-13T14:13:39,510 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42135 2024-11-13T14:13:39,528 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/eb1966fd6578:0 server-side Connection retries=45 2024-11-13T14:13:39,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:39,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:39,529 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T14:13:39,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T14:13:39,529 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T14:13:39,530 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T14:13:39,530 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T14:13:39,531 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41771 2024-11-13T14:13:39,533 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41771 connecting to ZooKeeper ensemble=127.0.0.1:59966 2024-11-13T14:13:39,535 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T14:13:39,538 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T14:13:39,554 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417710x0, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T14:13:39,555 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T14:13:39,555 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41771-0x101348b60910003 connected 2024-11-13T14:13:39,555 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T14:13:39,556 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T14:13:39,558 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T14:13:39,560 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T14:13:39,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41771 2024-11-13T14:13:39,561 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41771 2024-11-13T14:13:39,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41771 2024-11-13T14:13:39,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41771 2024-11-13T14:13:39,567 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41771 2024-11-13T14:13:39,581 DEBUG [M:0;eb1966fd6578:36813 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;eb1966fd6578:36813 2024-11-13T14:13:39,582 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/eb1966fd6578,36813,1731507218489 2024-11-13T14:13:39,596 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T14:13:39,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T14:13:39,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T14:13:39,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T14:13:39,599 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/eb1966fd6578,36813,1731507218489 2024-11-13T14:13:39,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T14:13:39,627 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T14:13:39,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:39,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:39,627 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:39,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T14:13:39,628 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:39,628 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T14:13:39,630 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/eb1966fd6578,36813,1731507218489 from backup master directory 2024-11-13T14:13:39,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T14:13:39,644 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T14:13:39,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/eb1966fd6578,36813,1731507218489 2024-11-13T14:13:39,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T14:13:39,644 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T14:13:39,645 WARN [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T14:13:39,646 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=eb1966fd6578,36813,1731507218489 2024-11-13T14:13:39,648 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T14:13:39,650 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T14:13:39,705 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/hbase.id] with ID: 0f3e8730-69ea-4035-b687-193cf5981ed6 2024-11-13T14:13:39,705 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/.tmp/hbase.id 2024-11-13T14:13:39,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741826_1002 (size=42) 2024-11-13T14:13:39,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741826_1002 (size=42) 2024-11-13T14:13:39,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741826_1002 (size=42) 2024-11-13T14:13:39,721 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/.tmp/hbase.id]:[hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/hbase.id] 2024-11-13T14:13:39,765 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T14:13:39,770 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T14:13:39,788 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-13T14:13:39,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:39,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:39,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:39,817 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:39,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741827_1003 (size=196) 2024-11-13T14:13:39,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741827_1003 (size=196) 2024-11-13T14:13:39,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741827_1003 (size=196) 2024-11-13T14:13:39,850 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T14:13:39,851 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T14:13:39,864 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:13:39,867 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T14:13:39,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741828_1004 (size=1189) 2024-11-13T14:13:39,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741828_1004 (size=1189) 2024-11-13T14:13:39,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741828_1004 (size=1189) 2024-11-13T14:13:39,914 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store 2024-11-13T14:13:39,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741829_1005 (size=34) 2024-11-13T14:13:39,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741829_1005 (size=34) 2024-11-13T14:13:39,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741829_1005 (size=34) 2024-11-13T14:13:39,939 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T14:13:39,942 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:39,944 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T14:13:39,944 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T14:13:39,944 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T14:13:39,946 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T14:13:39,946 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T14:13:39,946 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T14:13:39,947 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731507219943Disabling compacts and flushes for region at 1731507219943Disabling writes for close at 1731507219946 (+3 ms)Writing region close event to WAL at 1731507219946Closed at 1731507219946 2024-11-13T14:13:39,949 WARN [master/eb1966fd6578:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/.initializing 2024-11-13T14:13:39,949 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/WALs/eb1966fd6578,36813,1731507218489 2024-11-13T14:13:39,959 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T14:13:39,973 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eb1966fd6578%2C36813%2C1731507218489, suffix=, logDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/WALs/eb1966fd6578,36813,1731507218489, archiveDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/oldWALs, maxLogs=10 2024-11-13T14:13:39,994 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/WALs/eb1966fd6578,36813,1731507218489/eb1966fd6578%2C36813%2C1731507218489.1731507219977, exclude list is [], retry=0 2024-11-13T14:13:40,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-7d222e8a-09d0-41ad-8291-a32be5623236,DISK] 2024-11-13T14:13:40,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33015,DS-9b4f3d38-26ab-4b10-b6b8-307e24d8051e,DISK] 2024-11-13T14:13:40,012 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46747,DS-0b2f9f3b-9824-4059-870f-9bb615a80b01,DISK] 2024-11-13T14:13:40,014 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T14:13:40,048 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/WALs/eb1966fd6578,36813,1731507218489/eb1966fd6578%2C36813%2C1731507218489.1731507219977 2024-11-13T14:13:40,049 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39941:39941),(127.0.0.1/127.0.0.1:42835:42835),(127.0.0.1/127.0.0.1:43485:43485)] 2024-11-13T14:13:40,050 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T14:13:40,050 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:40,053 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,054 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,089 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,114 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T14:13:40,118 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:40,120 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:40,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T14:13:40,125 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:40,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:40,126 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,129 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T14:13:40,129 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:40,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:40,131 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,134 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T14:13:40,134 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:40,135 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:40,135 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,139 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,141 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,147 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,147 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,151 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T14:13:40,155 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T14:13:40,160 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:40,162 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74696699, jitterRate=0.11306755244731903}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T14:13:40,170 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731507220066Initializing all the Stores at 1731507220068 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731507220069 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507220069Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507220070 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507220070Cleaning up temporary data from old regions at 1731507220147 (+77 ms)Region opened successfully at 1731507220170 (+23 ms) 2024-11-13T14:13:40,171 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T14:13:40,207 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46d6a19e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eb1966fd6578/172.17.0.2:0 2024-11-13T14:13:40,233 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T14:13:40,242 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:40,242 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:40,245 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:40,246 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T14:13:40,251 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T14:13:40,251 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T14:13:40,274 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T14:13:40,282 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T14:13:40,360 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T14:13:40,365 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T14:13:40,368 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T14:13:40,381 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T14:13:40,383 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T14:13:40,387 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T14:13:40,395 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T14:13:40,397 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T14:13:40,406 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T14:13:40,427 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T14:13:40,437 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T14:13:40,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T14:13:40,448 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T14:13:40,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T14:13:40,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T14:13:40,448 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,449 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,453 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=eb1966fd6578,36813,1731507218489, sessionid=0x101348b60910000, setting cluster-up flag (Was=false) 2024-11-13T14:13:40,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,486 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,522 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T14:13:40,527 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eb1966fd6578,36813,1731507218489 2024-11-13T14:13:40,554 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,554 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:40,585 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T14:13:40,589 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=eb1966fd6578,36813,1731507218489 2024-11-13T14:13:40,597 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T14:13:40,662 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:40,671 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T14:13:40,672 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(746): ClusterId : 0f3e8730-69ea-4035-b687-193cf5981ed6 2024-11-13T14:13:40,672 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(746): ClusterId : 0f3e8730-69ea-4035-b687-193cf5981ed6 2024-11-13T14:13:40,672 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.HRegionServer(746): ClusterId : 0f3e8730-69ea-4035-b687-193cf5981ed6 2024-11-13T14:13:40,674 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T14:13:40,674 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T14:13:40,674 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T14:13:40,677 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T14:13:40,682 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: eb1966fd6578,36813,1731507218489 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T14:13:40,688 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T14:13:40,688 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T14:13:40,688 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T14:13:40,688 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T14:13:40,688 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T14:13:40,688 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T14:13:40,688 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/eb1966fd6578:0, corePoolSize=5, maxPoolSize=5 2024-11-13T14:13:40,689 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/eb1966fd6578:0, corePoolSize=5, maxPoolSize=5 2024-11-13T14:13:40,689 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/eb1966fd6578:0, corePoolSize=5, maxPoolSize=5 2024-11-13T14:13:40,689 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/eb1966fd6578:0, corePoolSize=5, maxPoolSize=5 2024-11-13T14:13:40,689 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/eb1966fd6578:0, corePoolSize=10, maxPoolSize=10 2024-11-13T14:13:40,689 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,689 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/eb1966fd6578:0, corePoolSize=2, maxPoolSize=2 2024-11-13T14:13:40,689 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,690 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507250690 2024-11-13T14:13:40,691 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T14:13:40,692 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T14:13:40,695 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:40,695 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T14:13:40,696 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T14:13:40,696 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T14:13:40,696 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T14:13:40,697 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T14:13:40,697 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,700 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T14:13:40,700 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:40,701 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T14:13:40,701 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T14:13:40,702 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T14:13:40,702 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T14:13:40,702 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T14:13:40,702 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T14:13:40,703 DEBUG [RS:1;eb1966fd6578:42135 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55e48eb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eb1966fd6578/172.17.0.2:0 2024-11-13T14:13:40,703 DEBUG [RS:2;eb1966fd6578:41771 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d5f9eaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eb1966fd6578/172.17.0.2:0 2024-11-13T14:13:40,703 DEBUG [RS:0;eb1966fd6578:38145 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1996e4c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=eb1966fd6578/172.17.0.2:0 2024-11-13T14:13:40,704 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T14:13:40,705 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T14:13:40,708 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/eb1966fd6578:0:becomeActiveMaster-HFileCleaner.large.0-1731507220706,5,FailOnTimeoutGroup] 2024-11-13T14:13:40,710 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/eb1966fd6578:0:becomeActiveMaster-HFileCleaner.small.0-1731507220709,5,FailOnTimeoutGroup] 2024-11-13T14:13:40,710 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,710 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T14:13:40,712 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,712 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,721 DEBUG [RS:1;eb1966fd6578:42135 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;eb1966fd6578:42135 2024-11-13T14:13:40,721 DEBUG [RS:2;eb1966fd6578:41771 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;eb1966fd6578:41771 2024-11-13T14:13:40,722 DEBUG [RS:0;eb1966fd6578:38145 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;eb1966fd6578:38145 2024-11-13T14:13:40,724 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T14:13:40,724 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T14:13:40,724 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T14:13:40,724 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T14:13:40,724 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T14:13:40,724 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T14:13:40,724 DEBUG [RS:0;eb1966fd6578:38145 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T14:13:40,724 DEBUG [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T14:13:40,724 DEBUG [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T14:13:40,727 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(2659): reportForDuty to master=eb1966fd6578,36813,1731507218489 with port=41771, startcode=1731507219528 2024-11-13T14:13:40,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741831_1007 (size=1321) 2024-11-13T14:13:40,727 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.HRegionServer(2659): reportForDuty to master=eb1966fd6578,36813,1731507218489 with port=38145, startcode=1731507219374 2024-11-13T14:13:40,727 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(2659): reportForDuty to master=eb1966fd6578,36813,1731507218489 with port=42135, startcode=1731507219475 2024-11-13T14:13:40,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741831_1007 (size=1321) 2024-11-13T14:13:40,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741831_1007 (size=1321) 2024-11-13T14:13:40,731 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T14:13:40,732 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:40,742 DEBUG [RS:0;eb1966fd6578:38145 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T14:13:40,742 DEBUG [RS:1;eb1966fd6578:42135 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T14:13:40,742 DEBUG [RS:2;eb1966fd6578:41771 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T14:13:40,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741832_1008 (size=32) 2024-11-13T14:13:40,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741832_1008 (size=32) 2024-11-13T14:13:40,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741832_1008 (size=32) 2024-11-13T14:13:40,751 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:40,754 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T14:13:40,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T14:13:40,758 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:40,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:40,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T14:13:40,762 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T14:13:40,763 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:40,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:40,764 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T14:13:40,769 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T14:13:40,769 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:40,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:40,771 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T14:13:40,774 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T14:13:40,774 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:40,775 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:40,775 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T14:13:40,777 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740 2024-11-13T14:13:40,778 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740 2024-11-13T14:13:40,779 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49471, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T14:13:40,779 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35873, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T14:13:40,779 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49483, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T14:13:40,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T14:13:40,781 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T14:13:40,782 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T14:13:40,785 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T14:13:40,787 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eb1966fd6578,38145,1731507219374 2024-11-13T14:13:40,789 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:40,789 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] master.ServerManager(517): Registering regionserver=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:40,790 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69202752, jitterRate=0.03120136260986328}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T14:13:40,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731507220751Initializing all the Stores at 1731507220753 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731507220753Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731507220754 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507220754Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731507220754Cleaning up temporary data from old regions at 1731507220781 (+27 ms)Region opened successfully at 1731507220795 (+14 ms) 2024-11-13T14:13:40,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T14:13:40,796 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T14:13:40,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T14:13:40,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T14:13:40,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T14:13:40,798 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T14:13:40,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731507220796Disabling compacts and flushes for region at 1731507220796Disabling writes for close at 1731507220796Writing region close event to WAL at 1731507220797 (+1 ms)Closed at 1731507220798 (+1 ms) 2024-11-13T14:13:40,801 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eb1966fd6578,41771,1731507219528 2024-11-13T14:13:40,801 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:40,801 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T14:13:40,802 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] master.ServerManager(517): Registering regionserver=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:40,805 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] master.ServerManager(363): Checking decommissioned status of RegionServer eb1966fd6578,42135,1731507219475 2024-11-13T14:13:40,805 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] master.ServerManager(517): Registering regionserver=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:40,806 DEBUG [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:40,806 DEBUG [RS:0;eb1966fd6578:38145 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:40,806 DEBUG [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34289 2024-11-13T14:13:40,806 DEBUG [RS:0;eb1966fd6578:38145 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34289 2024-11-13T14:13:40,806 DEBUG [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T14:13:40,806 DEBUG [RS:0;eb1966fd6578:38145 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T14:13:40,809 DEBUG [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:40,810 DEBUG [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34289 2024-11-13T14:13:40,810 DEBUG [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T14:13:40,811 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T14:13:40,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T14:13:40,822 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:40,825 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T14:13:40,858 DEBUG [RS:2;eb1966fd6578:41771 {}] zookeeper.ZKUtil(111): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/eb1966fd6578,41771,1731507219528 2024-11-13T14:13:40,858 WARN [RS:2;eb1966fd6578:41771 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T14:13:40,858 INFO [RS:2;eb1966fd6578:41771 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T14:13:40,858 DEBUG [RS:0;eb1966fd6578:38145 {}] zookeeper.ZKUtil(111): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/eb1966fd6578,38145,1731507219374 2024-11-13T14:13:40,859 DEBUG [RS:1;eb1966fd6578:42135 {}] zookeeper.ZKUtil(111): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/eb1966fd6578,42135,1731507219475 2024-11-13T14:13:40,859 WARN [RS:0;eb1966fd6578:38145 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T14:13:40,859 DEBUG [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,41771,1731507219528 2024-11-13T14:13:40,859 WARN [RS:1;eb1966fd6578:42135 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T14:13:40,859 INFO [RS:0;eb1966fd6578:38145 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T14:13:40,859 INFO [RS:1;eb1966fd6578:42135 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T14:13:40,859 DEBUG [RS:0;eb1966fd6578:38145 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374 2024-11-13T14:13:40,860 DEBUG [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,42135,1731507219475 2024-11-13T14:13:40,861 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eb1966fd6578,41771,1731507219528] 2024-11-13T14:13:40,861 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eb1966fd6578,38145,1731507219374] 2024-11-13T14:13:40,861 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [eb1966fd6578,42135,1731507219475] 2024-11-13T14:13:40,887 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T14:13:40,887 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T14:13:40,887 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T14:13:40,905 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T14:13:40,905 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T14:13:40,905 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T14:13:40,911 INFO [RS:0;eb1966fd6578:38145 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T14:13:40,911 INFO [RS:2;eb1966fd6578:41771 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T14:13:40,911 INFO [RS:1;eb1966fd6578:42135 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T14:13:40,912 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,912 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,912 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,913 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T14:13:40,913 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T14:13:40,913 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T14:13:40,918 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T14:13:40,918 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T14:13:40,918 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T14:13:40,920 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,920 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,920 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,921 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0, corePoolSize=2, maxPoolSize=2 2024-11-13T14:13:40,921 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0, corePoolSize=2, maxPoolSize=2 2024-11-13T14:13:40,921 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,921 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0, corePoolSize=2, maxPoolSize=2 2024-11-13T14:13:40,922 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eb1966fd6578:0, corePoolSize=3, maxPoolSize=3 2024-11-13T14:13:40,922 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:1;eb1966fd6578:42135 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eb1966fd6578:0, corePoolSize=3, maxPoolSize=3 2024-11-13T14:13:40,922 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,922 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/eb1966fd6578:0, corePoolSize=1, maxPoolSize=1 2024-11-13T14:13:40,923 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eb1966fd6578:0, corePoolSize=3, maxPoolSize=3 2024-11-13T14:13:40,923 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/eb1966fd6578:0, corePoolSize=3, maxPoolSize=3 2024-11-13T14:13:40,923 DEBUG [RS:0;eb1966fd6578:38145 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eb1966fd6578:0, corePoolSize=3, maxPoolSize=3 2024-11-13T14:13:40,923 DEBUG [RS:2;eb1966fd6578:41771 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/eb1966fd6578:0, corePoolSize=3, maxPoolSize=3 2024-11-13T14:13:40,927 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,927 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,927 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,927 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,928 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,928 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,42135,1731507219475-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T14:13:40,928 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,928 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,928 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,928 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,928 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,928 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,38145,1731507219374-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T14:13:40,930 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,930 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,930 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,930 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,930 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,931 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,41771,1731507219528-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T14:13:40,952 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T14:13:40,952 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T14:13:40,952 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T14:13:40,954 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,41771,1731507219528-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,954 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,38145,1731507219374-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,954 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,42135,1731507219475-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,954 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,954 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,954 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,955 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.Replication(171): eb1966fd6578,41771,1731507219528 started 2024-11-13T14:13:40,955 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.Replication(171): eb1966fd6578,38145,1731507219374 started 2024-11-13T14:13:40,955 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.Replication(171): eb1966fd6578,42135,1731507219475 started 2024-11-13T14:13:40,971 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,971 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(1482): Serving as eb1966fd6578,41771,1731507219528, RpcServer on eb1966fd6578/172.17.0.2:41771, sessionid=0x101348b60910003 2024-11-13T14:13:40,972 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T14:13:40,972 DEBUG [RS:2;eb1966fd6578:41771 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eb1966fd6578,41771,1731507219528 2024-11-13T14:13:40,972 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eb1966fd6578,41771,1731507219528' 2024-11-13T14:13:40,972 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T14:13:40,973 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T14:13:40,974 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T14:13:40,974 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T14:13:40,974 DEBUG [RS:2;eb1966fd6578:41771 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eb1966fd6578,41771,1731507219528 2024-11-13T14:13:40,974 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eb1966fd6578,41771,1731507219528' 2024-11-13T14:13:40,974 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T14:13:40,975 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T14:13:40,975 DEBUG [RS:2;eb1966fd6578:41771 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T14:13:40,975 INFO [RS:2;eb1966fd6578:41771 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T14:13:40,975 INFO [RS:2;eb1966fd6578:41771 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T14:13:40,976 WARN [eb1966fd6578:36813 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T14:13:40,977 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,977 INFO [RS:0;eb1966fd6578:38145 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:40,977 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(1482): Serving as eb1966fd6578,42135,1731507219475, RpcServer on eb1966fd6578/172.17.0.2:42135, sessionid=0x101348b60910002 2024-11-13T14:13:40,977 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.HRegionServer(1482): Serving as eb1966fd6578,38145,1731507219374, RpcServer on eb1966fd6578/172.17.0.2:38145, sessionid=0x101348b60910001 2024-11-13T14:13:40,977 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T14:13:40,977 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T14:13:40,978 DEBUG [RS:1;eb1966fd6578:42135 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eb1966fd6578,42135,1731507219475 2024-11-13T14:13:40,978 DEBUG [RS:0;eb1966fd6578:38145 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager eb1966fd6578,38145,1731507219374 2024-11-13T14:13:40,978 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eb1966fd6578,42135,1731507219475' 2024-11-13T14:13:40,978 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eb1966fd6578,38145,1731507219374' 2024-11-13T14:13:40,978 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T14:13:40,978 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T14:13:40,979 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T14:13:40,979 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T14:13:40,979 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T14:13:40,979 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T14:13:40,979 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T14:13:40,979 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T14:13:40,979 DEBUG [RS:1;eb1966fd6578:42135 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eb1966fd6578,42135,1731507219475 2024-11-13T14:13:40,980 DEBUG [RS:0;eb1966fd6578:38145 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager eb1966fd6578,38145,1731507219374 2024-11-13T14:13:40,980 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eb1966fd6578,42135,1731507219475' 2024-11-13T14:13:40,980 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'eb1966fd6578,38145,1731507219374' 2024-11-13T14:13:40,980 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T14:13:40,980 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T14:13:40,980 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T14:13:40,980 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T14:13:40,981 DEBUG [RS:0;eb1966fd6578:38145 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T14:13:40,981 INFO [RS:0;eb1966fd6578:38145 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T14:13:40,981 DEBUG [RS:1;eb1966fd6578:42135 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T14:13:40,981 INFO [RS:0;eb1966fd6578:38145 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T14:13:40,981 INFO [RS:1;eb1966fd6578:42135 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T14:13:40,981 INFO [RS:1;eb1966fd6578:42135 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T14:13:41,082 INFO [RS:0;eb1966fd6578:38145 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T14:13:41,082 INFO [RS:2;eb1966fd6578:41771 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T14:13:41,082 INFO [RS:1;eb1966fd6578:42135 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T14:13:41,085 INFO [RS:0;eb1966fd6578:38145 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eb1966fd6578%2C38145%2C1731507219374, suffix=, logDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374, archiveDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/oldWALs, maxLogs=32 2024-11-13T14:13:41,085 INFO [RS:2;eb1966fd6578:41771 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eb1966fd6578%2C41771%2C1731507219528, suffix=, logDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,41771,1731507219528, archiveDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/oldWALs, maxLogs=32 2024-11-13T14:13:41,085 INFO [RS:1;eb1966fd6578:42135 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eb1966fd6578%2C42135%2C1731507219475, suffix=, logDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,42135,1731507219475, archiveDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/oldWALs, maxLogs=32 2024-11-13T14:13:41,103 DEBUG [RS:2;eb1966fd6578:41771 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,41771,1731507219528/eb1966fd6578%2C41771%2C1731507219528.1731507221090, exclude list is [], retry=0 2024-11-13T14:13:41,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46747,DS-0b2f9f3b-9824-4059-870f-9bb615a80b01,DISK] 2024-11-13T14:13:41,108 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33015,DS-9b4f3d38-26ab-4b10-b6b8-307e24d8051e,DISK] 2024-11-13T14:13:41,133 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-7d222e8a-09d0-41ad-8291-a32be5623236,DISK] 2024-11-13T14:13:41,133 DEBUG [RS:0;eb1966fd6578:38145 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374/eb1966fd6578%2C38145%2C1731507219374.1731507221090, exclude list is [], retry=0 2024-11-13T14:13:41,133 DEBUG [RS:1;eb1966fd6578:42135 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,42135,1731507219475/eb1966fd6578%2C42135%2C1731507219475.1731507221090, exclude list is [], retry=0 2024-11-13T14:13:41,139 INFO [RS:2;eb1966fd6578:41771 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,41771,1731507219528/eb1966fd6578%2C41771%2C1731507219528.1731507221090 2024-11-13T14:13:41,140 DEBUG [RS:2;eb1966fd6578:41771 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39941:39941),(127.0.0.1/127.0.0.1:42835:42835),(127.0.0.1/127.0.0.1:43485:43485)] 2024-11-13T14:13:41,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-7d222e8a-09d0-41ad-8291-a32be5623236,DISK] 2024-11-13T14:13:41,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33015,DS-9b4f3d38-26ab-4b10-b6b8-307e24d8051e,DISK] 2024-11-13T14:13:41,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33015,DS-9b4f3d38-26ab-4b10-b6b8-307e24d8051e,DISK] 2024-11-13T14:13:41,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46747,DS-0b2f9f3b-9824-4059-870f-9bb615a80b01,DISK] 2024-11-13T14:13:41,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46747,DS-0b2f9f3b-9824-4059-870f-9bb615a80b01,DISK] 2024-11-13T14:13:41,142 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-7d222e8a-09d0-41ad-8291-a32be5623236,DISK] 2024-11-13T14:13:41,148 INFO [RS:1;eb1966fd6578:42135 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,42135,1731507219475/eb1966fd6578%2C42135%2C1731507219475.1731507221090 2024-11-13T14:13:41,149 INFO [RS:0;eb1966fd6578:38145 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374/eb1966fd6578%2C38145%2C1731507219374.1731507221090 2024-11-13T14:13:41,150 DEBUG [RS:1;eb1966fd6578:42135 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39941:39941),(127.0.0.1/127.0.0.1:42835:42835),(127.0.0.1/127.0.0.1:43485:43485)] 2024-11-13T14:13:41,150 DEBUG [RS:0;eb1966fd6578:38145 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43485:43485),(127.0.0.1/127.0.0.1:39941:39941),(127.0.0.1/127.0.0.1:42835:42835)] 2024-11-13T14:13:41,231 DEBUG [eb1966fd6578:36813 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T14:13:41,240 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(204): Hosts are {eb1966fd6578=0} racks are {/default-rack=0} 2024-11-13T14:13:41,246 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T14:13:41,246 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T14:13:41,246 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T14:13:41,246 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T14:13:41,247 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T14:13:41,247 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T14:13:41,247 INFO [eb1966fd6578:36813 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T14:13:41,247 INFO [eb1966fd6578:36813 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T14:13:41,247 INFO [eb1966fd6578:36813 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T14:13:41,247 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T14:13:41,253 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:41,260 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,38145,1731507219374, state=OPENING 2024-11-13T14:13:41,318 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T14:13:41,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:41,329 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:41,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:41,329 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:41,331 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:41,331 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:41,332 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:41,332 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:41,335 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:41,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,38145,1731507219374}] 2024-11-13T14:13:41,515 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T14:13:41,518 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56449, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T14:13:41,531 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T14:13:41,531 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T14:13:41,532 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T14:13:41,535 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eb1966fd6578%2C38145%2C1731507219374.meta, suffix=.meta, logDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374, archiveDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/oldWALs, maxLogs=32 2024-11-13T14:13:41,550 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, exclude list is [], retry=0 2024-11-13T14:13:41,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-7d222e8a-09d0-41ad-8291-a32be5623236,DISK] 2024-11-13T14:13:41,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46747,DS-0b2f9f3b-9824-4059-870f-9bb615a80b01,DISK] 2024-11-13T14:13:41,554 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33015,DS-9b4f3d38-26ab-4b10-b6b8-307e24d8051e,DISK] 2024-11-13T14:13:41,557 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta 2024-11-13T14:13:41,557 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43485:43485),(127.0.0.1/127.0.0.1:42835:42835),(127.0.0.1/127.0.0.1:39941:39941)] 2024-11-13T14:13:41,557 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T14:13:41,559 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T14:13:41,561 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T14:13:41,566 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T14:13:41,569 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T14:13:41,570 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:41,570 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T14:13:41,570 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T14:13:41,573 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T14:13:41,575 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T14:13:41,575 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:41,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:41,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T14:13:41,578 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T14:13:41,578 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:41,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:41,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T14:13:41,580 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T14:13:41,580 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:41,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:41,581 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T14:13:41,583 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T14:13:41,583 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:41,584 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:41,584 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T14:13:41,585 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740 2024-11-13T14:13:41,588 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740 2024-11-13T14:13:41,591 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T14:13:41,591 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T14:13:41,592 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T14:13:41,595 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T14:13:41,597 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59349146, jitterRate=-0.11562880873680115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T14:13:41,597 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T14:13:41,598 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731507221570Writing region info on filesystem at 1731507221571 (+1 ms)Initializing all the Stores at 1731507221573 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731507221573Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731507221573Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507221573Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731507221573Cleaning up temporary data from old regions at 1731507221591 (+18 ms)Running coprocessor post-open hooks at 1731507221597 (+6 ms)Region opened successfully at 1731507221598 (+1 ms) 2024-11-13T14:13:41,605 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731507221508 2024-11-13T14:13:41,617 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T14:13:41,617 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T14:13:41,622 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:41,624 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,38145,1731507219374, state=OPEN 2024-11-13T14:13:41,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:41,638 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:41,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:41,638 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:41,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:41,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:41,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:41,638 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:41,638 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:41,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T14:13:41,645 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,38145,1731507219374 in 302 msec 2024-11-13T14:13:41,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T14:13:41,654 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 837 msec 2024-11-13T14:13:41,655 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:41,655 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T14:13:41,674 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T14:13:41,676 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1] 2024-11-13T14:13:41,697 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T14:13:41,699 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57651, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T14:13:41,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0940 sec 2024-11-13T14:13:41,722 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731507221722, completionTime=-1 2024-11-13T14:13:41,725 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T14:13:41,726 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:41,753 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-13T14:13:41,753 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507281753 2024-11-13T14:13:41,753 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507341753 2024-11-13T14:13:41,753 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 27 msec 2024-11-13T14:13:41,755 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:41,762 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,36813,1731507218489-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:41,762 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,36813,1731507218489-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:41,762 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,36813,1731507218489-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:41,764 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-eb1966fd6578:36813, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:41,764 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:41,765 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:41,771 DEBUG [master/eb1966fd6578:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T14:13:41,794 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.148sec 2024-11-13T14:13:41,795 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T14:13:41,797 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T14:13:41,798 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T14:13:41,798 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T14:13:41,798 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T14:13:41,799 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,36813,1731507218489-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T14:13:41,799 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,36813,1731507218489-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T14:13:41,804 DEBUG [master/eb1966fd6578:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T14:13:41,805 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T14:13:41,805 INFO [master/eb1966fd6578:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=eb1966fd6578,36813,1731507218489-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T14:13:41,887 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41247b4f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T14:13:41,892 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T14:13:41,892 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T14:13:41,896 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request eb1966fd6578,36813,-1 for getting cluster id 2024-11-13T14:13:41,899 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T14:13:41,908 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '0f3e8730-69ea-4035-b687-193cf5981ed6' 2024-11-13T14:13:41,910 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T14:13:41,911 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "0f3e8730-69ea-4035-b687-193cf5981ed6" 2024-11-13T14:13:41,914 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44f07c25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T14:13:41,914 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [eb1966fd6578,36813,-1] 2024-11-13T14:13:41,916 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T14:13:41,919 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T14:13:41,919 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36552, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T14:13:41,922 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d07ca3f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T14:13:41,922 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T14:13:41,929 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1] 2024-11-13T14:13:41,930 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T14:13:41,932 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37956, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T14:13:41,954 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=eb1966fd6578,36813,1731507218489 2024-11-13T14:13:41,958 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T14:13:41,983 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is eb1966fd6578,36813,1731507218489 2024-11-13T14:13:41,986 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@47d5c71e 2024-11-13T14:13:41,988 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T14:13:41,991 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36554, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T14:13:41,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T14:13:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:42,018 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T14:13:42,020 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-13T14:13:42,020 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:42,024 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T14:13:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T14:13:42,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741837_1013 (size=358) 2024-11-13T14:13:42,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741837_1013 (size=358) 2024-11-13T14:13:42,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741837_1013 (size=358) 2024-11-13T14:13:42,047 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => b2697dde157dfed90cc8595ff82dc081, NAME => 'test,,1731507221993.b2697dde157dfed90cc8595ff82dc081.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,047 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => d2b53404c106b2f9960c2de982a64c06, NAME => 'test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,049 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 0e888576e09ecc86a3b2198a9e11c7b4, NAME => 'test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741838_1014 (size=42) 2024-11-13T14:13:42,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741840_1016 (size=45) 2024-11-13T14:13:42,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741840_1016 (size=45) 2024-11-13T14:13:42,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741840_1016 (size=45) 2024-11-13T14:13:42,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741838_1014 (size=42) 2024-11-13T14:13:42,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741838_1014 (size=42) 2024-11-13T14:13:42,077 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731507221993.b2697dde157dfed90cc8595ff82dc081.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,078 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 0e888576e09ecc86a3b2198a9e11c7b4, disabling compactions & flushes 2024-11-13T14:13:42,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing b2697dde157dfed90cc8595ff82dc081, disabling compactions & flushes 2024-11-13T14:13:42,078 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:42,078 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:42,078 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:42,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:42,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. after waiting 0 ms 2024-11-13T14:13:42,078 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. after waiting 0 ms 2024-11-13T14:13:42,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:42,078 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:42,078 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:42,078 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:42,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for b2697dde157dfed90cc8595ff82dc081: Waiting for close lock at 1731507222078Disabling compacts and flushes for region at 1731507222078Disabling writes for close at 1731507222078Writing region close event to WAL at 1731507222078Closed at 1731507222078 2024-11-13T14:13:42,078 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 0e888576e09ecc86a3b2198a9e11c7b4: Waiting for close lock at 1731507222078Disabling compacts and flushes for region at 1731507222078Disabling writes for close at 1731507222078Writing region close event to WAL at 1731507222078Closed at 1731507222078 2024-11-13T14:13:42,079 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bc9bead55355bc7d07a869ba01fa7404, NAME => 'test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,079 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => eb1bb70b6e4e99eeece01c54dd8a665c, NAME => 'test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741839_1015 (size=45) 2024-11-13T14:13:42,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741839_1015 (size=45) 2024-11-13T14:13:42,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741839_1015 (size=45) 2024-11-13T14:13:42,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741841_1017 (size=45) 2024-11-13T14:13:42,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741841_1017 (size=45) 2024-11-13T14:13:42,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741841_1017 (size=45) 2024-11-13T14:13:42,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741842_1018 (size=45) 2024-11-13T14:13:42,107 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741842_1018 (size=45) 2024-11-13T14:13:42,107 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing eb1bb70b6e4e99eeece01c54dd8a665c, disabling compactions & flushes 2024-11-13T14:13:42,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741842_1018 (size=45) 2024-11-13T14:13:42,107 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:42,107 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:42,107 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. after waiting 0 ms 2024-11-13T14:13:42,107 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:42,107 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:42,107 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for eb1bb70b6e4e99eeece01c54dd8a665c: Waiting for close lock at 1731507222107Disabling compacts and flushes for region at 1731507222107Disabling writes for close at 1731507222107Writing region close event to WAL at 1731507222107Closed at 1731507222107 2024-11-13T14:13:42,108 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 17aeba8ae6d5f57b63d67825a02878d1, NAME => 'test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,108 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing bc9bead55355bc7d07a869ba01fa7404, disabling compactions & flushes 2024-11-13T14:13:42,109 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:42,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:42,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. after waiting 0 ms 2024-11-13T14:13:42,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:42,109 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:42,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for bc9bead55355bc7d07a869ba01fa7404: Waiting for close lock at 1731507222109Disabling compacts and flushes for region at 1731507222109Disabling writes for close at 1731507222109Writing region close event to WAL at 1731507222109Closed at 1731507222109 2024-11-13T14:13:42,110 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => def71812430057c99b7be5281021b22c, NAME => 'test,ggg,1731507221993.def71812430057c99b7be5281021b22c.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741843_1019 (size=45) 2024-11-13T14:13:42,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741844_1020 (size=45) 2024-11-13T14:13:42,129 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741844_1020 (size=45) 2024-11-13T14:13:42,129 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 17aeba8ae6d5f57b63d67825a02878d1, disabling compactions & flushes 2024-11-13T14:13:42,129 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:42,129 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:42,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741844_1020 (size=45) 2024-11-13T14:13:42,130 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. after waiting 0 ms 2024-11-13T14:13:42,130 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:42,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741843_1019 (size=45) 2024-11-13T14:13:42,130 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:42,130 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 17aeba8ae6d5f57b63d67825a02878d1: Waiting for close lock at 1731507222129Disabling compacts and flushes for region at 1731507222129Disabling writes for close at 1731507222130 (+1 ms)Writing region close event to WAL at 1731507222130Closed at 1731507222130 2024-11-13T14:13:42,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741843_1019 (size=45) 2024-11-13T14:13:42,130 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ggg,1731507221993.def71812430057c99b7be5281021b22c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,131 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing def71812430057c99b7be5281021b22c, disabling compactions & flushes 2024-11-13T14:13:42,131 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 668c3a8f00f6512df533713b2c2d46cb, NAME => 'test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,131 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:42,131 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:42,131 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731507221993.def71812430057c99b7be5281021b22c. after waiting 0 ms 2024-11-13T14:13:42,131 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:42,131 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:42,131 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for def71812430057c99b7be5281021b22c: Waiting for close lock at 1731507222131Disabling compacts and flushes for region at 1731507222131Disabling writes for close at 1731507222131Writing region close event to WAL at 1731507222131Closed at 1731507222131 2024-11-13T14:13:42,132 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dbe405b0820b2bbf65d6f5d084e654ed, NAME => 'test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T14:13:42,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741845_1021 (size=45) 2024-11-13T14:13:42,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741846_1022 (size=45) 2024-11-13T14:13:42,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741846_1022 (size=45) 2024-11-13T14:13:42,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741846_1022 (size=45) 2024-11-13T14:13:42,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741845_1021 (size=45) 2024-11-13T14:13:42,153 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741845_1021 (size=45) 2024-11-13T14:13:42,153 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 668c3a8f00f6512df533713b2c2d46cb, disabling compactions & flushes 2024-11-13T14:13:42,153 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:42,153 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:42,153 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. after waiting 0 ms 2024-11-13T14:13:42,153 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:42,153 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:42,153 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 668c3a8f00f6512df533713b2c2d46cb: Waiting for close lock at 1731507222153Disabling compacts and flushes for region at 1731507222153Disabling writes for close at 1731507222153Writing region close event to WAL at 1731507222153Closed at 1731507222153 2024-11-13T14:13:42,154 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ef7ba06bec150589d1c840bbd1b13d9b, NAME => 'test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,154 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,154 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing dbe405b0820b2bbf65d6f5d084e654ed, disabling compactions & flushes 2024-11-13T14:13:42,155 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:42,155 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:42,155 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. after waiting 0 ms 2024-11-13T14:13:42,155 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:42,155 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:42,155 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for dbe405b0820b2bbf65d6f5d084e654ed: Waiting for close lock at 1731507222154Disabling compacts and flushes for region at 1731507222154Disabling writes for close at 1731507222155 (+1 ms)Writing region close event to WAL at 1731507222155Closed at 1731507222155 2024-11-13T14:13:42,155 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 409ba30f82abbec05bb6c84a023d1e9a, NAME => 'test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741847_1023 (size=45) 2024-11-13T14:13:42,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741847_1023 (size=45) 2024-11-13T14:13:42,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741847_1023 (size=45) 2024-11-13T14:13:42,168 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,169 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ef7ba06bec150589d1c840bbd1b13d9b, disabling compactions & flushes 2024-11-13T14:13:42,169 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:42,169 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:42,169 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. after waiting 0 ms 2024-11-13T14:13:42,169 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:42,169 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:42,169 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ef7ba06bec150589d1c840bbd1b13d9b: Waiting for close lock at 1731507222168Disabling compacts and flushes for region at 1731507222169 (+1 ms)Disabling writes for close at 1731507222169Writing region close event to WAL at 1731507222169Closed at 1731507222169 2024-11-13T14:13:42,170 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => cb7d9e65213673705aa2d05e9c49ef5f, NAME => 'test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741848_1024 (size=45) 2024-11-13T14:13:42,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741848_1024 (size=45) 2024-11-13T14:13:42,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741848_1024 (size=45) 2024-11-13T14:13:42,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 409ba30f82abbec05bb6c84a023d1e9a, disabling compactions & flushes 2024-11-13T14:13:42,176 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:42,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:42,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. after waiting 0 ms 2024-11-13T14:13:42,177 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:42,177 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:42,177 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 409ba30f82abbec05bb6c84a023d1e9a: Waiting for close lock at 1731507222176Disabling compacts and flushes for region at 1731507222176Disabling writes for close at 1731507222177 (+1 ms)Writing region close event to WAL at 1731507222177Closed at 1731507222177 2024-11-13T14:13:42,178 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => edbfee0cdbcdc3896eb3a9b618149bc3, NAME => 'test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741849_1025 (size=45) 2024-11-13T14:13:42,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741849_1025 (size=45) 2024-11-13T14:13:42,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741849_1025 (size=45) 2024-11-13T14:13:42,185 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,185 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing cb7d9e65213673705aa2d05e9c49ef5f, disabling compactions & flushes 2024-11-13T14:13:42,185 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:42,185 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:42,185 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. after waiting 0 ms 2024-11-13T14:13:42,186 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:42,186 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:42,186 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for cb7d9e65213673705aa2d05e9c49ef5f: Waiting for close lock at 1731507222185Disabling compacts and flushes for region at 1731507222185Disabling writes for close at 1731507222185Writing region close event to WAL at 1731507222186 (+1 ms)Closed at 1731507222186 2024-11-13T14:13:42,187 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 026ea48c246e26b4197a66fa854f5022, NAME => 'test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741850_1026 (size=45) 2024-11-13T14:13:42,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741850_1026 (size=45) 2024-11-13T14:13:42,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741850_1026 (size=45) 2024-11-13T14:13:42,197 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,198 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing edbfee0cdbcdc3896eb3a9b618149bc3, disabling compactions & flushes 2024-11-13T14:13:42,198 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:42,198 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:42,198 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. after waiting 0 ms 2024-11-13T14:13:42,198 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:42,198 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:42,198 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for edbfee0cdbcdc3896eb3a9b618149bc3: Waiting for close lock at 1731507222197Disabling compacts and flushes for region at 1731507222197Disabling writes for close at 1731507222198 (+1 ms)Writing region close event to WAL at 1731507222198Closed at 1731507222198 2024-11-13T14:13:42,199 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 087abf0ac7c707f0f540d20925d9d683, NAME => 'test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741851_1027 (size=45) 2024-11-13T14:13:42,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741851_1027 (size=45) 2024-11-13T14:13:42,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741851_1027 (size=45) 2024-11-13T14:13:42,206 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,206 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 026ea48c246e26b4197a66fa854f5022, disabling compactions & flushes 2024-11-13T14:13:42,206 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:42,206 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:42,206 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. after waiting 0 ms 2024-11-13T14:13:42,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:42,207 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:42,207 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 026ea48c246e26b4197a66fa854f5022: Waiting for close lock at 1731507222206Disabling compacts and flushes for region at 1731507222206Disabling writes for close at 1731507222207 (+1 ms)Writing region close event to WAL at 1731507222207Closed at 1731507222207 2024-11-13T14:13:42,207 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1d3783ebc90fc59392fa002c53f7c56f, NAME => 'test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741852_1028 (size=45) 2024-11-13T14:13:42,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741852_1028 (size=45) 2024-11-13T14:13:42,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741852_1028 (size=45) 2024-11-13T14:13:42,217 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,217 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 087abf0ac7c707f0f540d20925d9d683, disabling compactions & flushes 2024-11-13T14:13:42,218 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:42,218 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:42,218 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. after waiting 0 ms 2024-11-13T14:13:42,218 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:42,218 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:42,218 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 087abf0ac7c707f0f540d20925d9d683: Waiting for close lock at 1731507222217Disabling compacts and flushes for region at 1731507222217Disabling writes for close at 1731507222218 (+1 ms)Writing region close event to WAL at 1731507222218Closed at 1731507222218 2024-11-13T14:13:42,219 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1f231052a3b0401ad15e94c5bf55a972, NAME => 'test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741853_1029 (size=45) 2024-11-13T14:13:42,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741853_1029 (size=45) 2024-11-13T14:13:42,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741853_1029 (size=45) 2024-11-13T14:13:42,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1d3783ebc90fc59392fa002c53f7c56f, disabling compactions & flushes 2024-11-13T14:13:42,227 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:42,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:42,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. after waiting 0 ms 2024-11-13T14:13:42,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:42,227 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:42,227 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1d3783ebc90fc59392fa002c53f7c56f: Waiting for close lock at 1731507222227Disabling compacts and flushes for region at 1731507222227Disabling writes for close at 1731507222227Writing region close event to WAL at 1731507222227Closed at 1731507222227 2024-11-13T14:13:42,228 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e501f0ec953c86735bbe2081ac738865, NAME => 'test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741854_1030 (size=45) 2024-11-13T14:13:42,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741854_1030 (size=45) 2024-11-13T14:13:42,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741854_1030 (size=45) 2024-11-13T14:13:42,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 1f231052a3b0401ad15e94c5bf55a972, disabling compactions & flushes 2024-11-13T14:13:42,233 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:42,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:42,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. after waiting 0 ms 2024-11-13T14:13:42,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:42,233 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:42,233 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1f231052a3b0401ad15e94c5bf55a972: Waiting for close lock at 1731507222233Disabling compacts and flushes for region at 1731507222233Disabling writes for close at 1731507222233Writing region close event to WAL at 1731507222233Closed at 1731507222233 2024-11-13T14:13:42,234 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => edf3600642cf1af12a36dc726f301ef2, NAME => 'test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741855_1031 (size=45) 2024-11-13T14:13:42,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741855_1031 (size=45) 2024-11-13T14:13:42,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741855_1031 (size=45) 2024-11-13T14:13:42,243 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,243 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e501f0ec953c86735bbe2081ac738865, disabling compactions & flushes 2024-11-13T14:13:42,243 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:42,243 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:42,243 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. after waiting 0 ms 2024-11-13T14:13:42,243 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:42,243 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:42,243 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e501f0ec953c86735bbe2081ac738865: Waiting for close lock at 1731507222243Disabling compacts and flushes for region at 1731507222243Disabling writes for close at 1731507222243Writing region close event to WAL at 1731507222243Closed at 1731507222243 2024-11-13T14:13:42,244 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => dd3915f2bd1ed9356a65d71ba154c88b, NAME => 'test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741856_1032 (size=45) 2024-11-13T14:13:42,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741856_1032 (size=45) 2024-11-13T14:13:42,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741856_1032 (size=45) 2024-11-13T14:13:42,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741857_1033 (size=45) 2024-11-13T14:13:42,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741857_1033 (size=45) 2024-11-13T14:13:42,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741857_1033 (size=45) 2024-11-13T14:13:42,269 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,270 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing dd3915f2bd1ed9356a65d71ba154c88b, disabling compactions & flushes 2024-11-13T14:13:42,270 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:42,270 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:42,270 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. after waiting 0 ms 2024-11-13T14:13:42,270 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:42,270 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:42,270 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for dd3915f2bd1ed9356a65d71ba154c88b: Waiting for close lock at 1731507222270Disabling compacts and flushes for region at 1731507222270Disabling writes for close at 1731507222270Writing region close event to WAL at 1731507222270Closed at 1731507222270 2024-11-13T14:13:42,271 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8ebd6e15e1eb3717f4dd8f1695d2b691, NAME => 'test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741858_1034 (size=45) 2024-11-13T14:13:42,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741858_1034 (size=45) 2024-11-13T14:13:42,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741858_1034 (size=45) 2024-11-13T14:13:42,281 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,281 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8ebd6e15e1eb3717f4dd8f1695d2b691, disabling compactions & flushes 2024-11-13T14:13:42,281 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:42,281 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:42,281 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. after waiting 0 ms 2024-11-13T14:13:42,281 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:42,281 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:42,282 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8ebd6e15e1eb3717f4dd8f1695d2b691: Waiting for close lock at 1731507222281Disabling compacts and flushes for region at 1731507222281Disabling writes for close at 1731507222281Writing region close event to WAL at 1731507222281Closed at 1731507222281 2024-11-13T14:13:42,282 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 24857182eb0da41497c7febd6e0c060d, NAME => 'test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741859_1035 (size=45) 2024-11-13T14:13:42,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741859_1035 (size=45) 2024-11-13T14:13:42,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741859_1035 (size=45) 2024-11-13T14:13:42,293 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,293 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 24857182eb0da41497c7febd6e0c060d, disabling compactions & flushes 2024-11-13T14:13:42,293 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:42,293 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:42,293 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. after waiting 0 ms 2024-11-13T14:13:42,293 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:42,293 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:42,293 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 24857182eb0da41497c7febd6e0c060d: Waiting for close lock at 1731507222293Disabling compacts and flushes for region at 1731507222293Disabling writes for close at 1731507222293Writing region close event to WAL at 1731507222293Closed at 1731507222293 2024-11-13T14:13:42,294 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2027122fd77a73d52ce4d3368dd23e0a, NAME => 'test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741860_1036 (size=45) 2024-11-13T14:13:42,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741860_1036 (size=45) 2024-11-13T14:13:42,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741860_1036 (size=45) 2024-11-13T14:13:42,304 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,304 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 2027122fd77a73d52ce4d3368dd23e0a, disabling compactions & flushes 2024-11-13T14:13:42,304 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:42,304 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:42,304 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. after waiting 0 ms 2024-11-13T14:13:42,304 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:42,304 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:42,304 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2027122fd77a73d52ce4d3368dd23e0a: Waiting for close lock at 1731507222304Disabling compacts and flushes for region at 1731507222304Disabling writes for close at 1731507222304Writing region close event to WAL at 1731507222304Closed at 1731507222304 2024-11-13T14:13:42,305 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 4db6e838ae3d565bc2af54adcdf3a3c0, NAME => 'test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741861_1037 (size=45) 2024-11-13T14:13:42,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741861_1037 (size=45) 2024-11-13T14:13:42,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741861_1037 (size=45) 2024-11-13T14:13:42,316 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,316 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 4db6e838ae3d565bc2af54adcdf3a3c0, disabling compactions & flushes 2024-11-13T14:13:42,316 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:42,316 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:42,316 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. after waiting 0 ms 2024-11-13T14:13:42,316 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:42,316 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:42,316 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 4db6e838ae3d565bc2af54adcdf3a3c0: Waiting for close lock at 1731507222316Disabling compacts and flushes for region at 1731507222316Disabling writes for close at 1731507222316Writing region close event to WAL at 1731507222316Closed at 1731507222316 2024-11-13T14:13:42,317 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9bab6e4180453738e691368cc5fbb429, NAME => 'test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741862_1038 (size=45) 2024-11-13T14:13:42,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741862_1038 (size=45) 2024-11-13T14:13:42,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741862_1038 (size=45) 2024-11-13T14:13:42,330 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,330 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9bab6e4180453738e691368cc5fbb429, disabling compactions & flushes 2024-11-13T14:13:42,330 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:42,330 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:42,330 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. after waiting 0 ms 2024-11-13T14:13:42,330 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:42,330 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:42,330 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9bab6e4180453738e691368cc5fbb429: Waiting for close lock at 1731507222330Disabling compacts and flushes for region at 1731507222330Disabling writes for close at 1731507222330Writing region close event to WAL at 1731507222330Closed at 1731507222330 2024-11-13T14:13:42,331 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ee5b69e1e01798b1055c61a7885f87b4, NAME => 'test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b 2024-11-13T14:13:42,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741863_1039 (size=42) 2024-11-13T14:13:42,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741863_1039 (size=42) 2024-11-13T14:13:42,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741863_1039 (size=42) 2024-11-13T14:13:42,341 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,341 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ee5b69e1e01798b1055c61a7885f87b4, disabling compactions & flushes 2024-11-13T14:13:42,341 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:42,341 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:42,341 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. after waiting 0 ms 2024-11-13T14:13:42,341 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:42,341 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:42,341 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ee5b69e1e01798b1055c61a7885f87b4: Waiting for close lock at 1731507222341Disabling compacts and flushes for region at 1731507222341Disabling writes for close at 1731507222341Writing region close event to WAL at 1731507222341Closed at 1731507222341 2024-11-13T14:13:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T14:13:42,488 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,488 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing d2b53404c106b2f9960c2de982a64c06, disabling compactions & flushes 2024-11-13T14:13:42,488 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:42,488 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:42,488 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. after waiting 0 ms 2024-11-13T14:13:42,488 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:42,488 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:42,489 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for d2b53404c106b2f9960c2de982a64c06: Waiting for close lock at 1731507222488Disabling compacts and flushes for region at 1731507222488Disabling writes for close at 1731507222488Writing region close event to WAL at 1731507222488Closed at 1731507222488 2024-11-13T14:13:42,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T14:13:42,658 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:42,658 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing edf3600642cf1af12a36dc726f301ef2, disabling compactions & flushes 2024-11-13T14:13:42,658 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:42,659 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:42,659 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. after waiting 0 ms 2024-11-13T14:13:42,659 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:42,659 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:42,659 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for edf3600642cf1af12a36dc726f301ef2: Waiting for close lock at 1731507222658Disabling compacts and flushes for region at 1731507222658Disabling writes for close at 1731507222659 (+1 ms)Writing region close event to WAL at 1731507222659Closed at 1731507222659 2024-11-13T14:13:42,662 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T14:13:42,669 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731507221993.b2697dde157dfed90cc8595ff82dc081.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,669 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,669 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,670 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,670 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,670 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731507221993.def71812430057c99b7be5281021b22c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,670 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,670 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,670 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,671 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,671 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,671 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,671 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,671 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,671 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,672 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,672 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,672 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,672 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,672 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,672 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,672 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,673 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,673 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,673 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,673 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731507222662"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731507222662"}]},"ts":"1731507222662"} 2024-11-13T14:13:42,720 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-13T14:13:42,722 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T14:13:42,724 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731507222722"}]},"ts":"1731507222722"} 2024-11-13T14:13:42,729 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-13T14:13:42,730 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {eb1966fd6578=0} racks are {/default-rack=0} 2024-11-13T14:13:42,746 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T14:13:42,746 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T14:13:42,746 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T14:13:42,746 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T14:13:42,746 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T14:13:42,746 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T14:13:42,746 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T14:13:42,746 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T14:13:42,746 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T14:13:42,746 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T14:13:42,749 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b2697dde157dfed90cc8595ff82dc081, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2b53404c106b2f9960c2de982a64c06, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e888576e09ecc86a3b2198a9e11c7b4, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bc9bead55355bc7d07a869ba01fa7404, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17aeba8ae6d5f57b63d67825a02878d1, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=668c3a8f00f6512df533713b2c2d46cb, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dbe405b0820b2bbf65d6f5d084e654ed, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=409ba30f82abbec05bb6c84a023d1e9a, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cb7d9e65213673705aa2d05e9c49ef5f, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=026ea48c246e26b4197a66fa854f5022, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=087abf0ac7c707f0f540d20925d9d683, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1f231052a3b0401ad15e94c5bf55a972, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e501f0ec953c86735bbe2081ac738865, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edf3600642cf1af12a36dc726f301ef2, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ebd6e15e1eb3717f4dd8f1695d2b691, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=24857182eb0da41497c7febd6e0c060d, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4db6e838ae3d565bc2af54adcdf3a3c0, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9bab6e4180453738e691368cc5fbb429, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN}] 2024-11-13T14:13:42,760 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:42,762 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN; state=OFFLINE, location=eb1966fd6578,38145,1731507219374; forceNewPlan=false, retain=false 2024-11-13T14:13:42,764 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9bab6e4180453738e691368cc5fbb429, ASSIGN 2024-11-13T14:13:42,765 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9bab6e4180453738e691368cc5fbb429, ASSIGN; state=OFFLINE, location=eb1966fd6578,41771,1731507219528; forceNewPlan=false, retain=false 2024-11-13T14:13:42,767 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=4db6e838ae3d565bc2af54adcdf3a3c0, ASSIGN 2024-11-13T14:13:42,768 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=4db6e838ae3d565bc2af54adcdf3a3c0, ASSIGN; state=OFFLINE, location=eb1966fd6578,42135,1731507219475; forceNewPlan=false, retain=false 2024-11-13T14:13:42,770 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:42,771 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN; state=OFFLINE, location=eb1966fd6578,38145,1731507219374; forceNewPlan=false, retain=false 2024-11-13T14:13:42,772 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=24857182eb0da41497c7febd6e0c060d, ASSIGN 2024-11-13T14:13:42,774 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=24857182eb0da41497c7febd6e0c060d, ASSIGN; state=OFFLINE, location=eb1966fd6578,41771,1731507219528; forceNewPlan=false, retain=false 2024-11-13T14:13:42,775 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8ebd6e15e1eb3717f4dd8f1695d2b691, ASSIGN 2024-11-13T14:13:42,776 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8ebd6e15e1eb3717f4dd8f1695d2b691, ASSIGN; state=OFFLINE, location=eb1966fd6578,42135,1731507219475; forceNewPlan=false, retain=false 2024-11-13T14:13:42,778 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:42,779 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN; state=OFFLINE, location=eb1966fd6578,38145,1731507219374; forceNewPlan=false, retain=false 2024-11-13T14:13:42,781 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edf3600642cf1af12a36dc726f301ef2, ASSIGN 2024-11-13T14:13:42,782 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=edf3600642cf1af12a36dc726f301ef2, ASSIGN; state=OFFLINE, location=eb1966fd6578,41771,1731507219528; forceNewPlan=false, retain=false 2024-11-13T14:13:42,783 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e501f0ec953c86735bbe2081ac738865, ASSIGN 2024-11-13T14:13:42,785 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e501f0ec953c86735bbe2081ac738865, ASSIGN; state=OFFLINE, location=eb1966fd6578,41771,1731507219528; forceNewPlan=false, retain=false 2024-11-13T14:13:42,786 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1f231052a3b0401ad15e94c5bf55a972, ASSIGN 2024-11-13T14:13:42,788 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1f231052a3b0401ad15e94c5bf55a972, ASSIGN; state=OFFLINE, location=eb1966fd6578,42135,1731507219475; forceNewPlan=false, retain=false 2024-11-13T14:13:42,789 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:42,791 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN; state=OFFLINE, location=eb1966fd6578,38145,1731507219374; forceNewPlan=false, retain=false 2024-11-13T14:13:42,793 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=087abf0ac7c707f0f540d20925d9d683, ASSIGN 2024-11-13T14:13:42,794 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=087abf0ac7c707f0f540d20925d9d683, ASSIGN; state=OFFLINE, location=eb1966fd6578,41771,1731507219528; forceNewPlan=false, retain=false 2024-11-13T14:13:42,795 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=026ea48c246e26b4197a66fa854f5022, ASSIGN 2024-11-13T14:13:42,797 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=026ea48c246e26b4197a66fa854f5022, ASSIGN; state=OFFLINE, location=eb1966fd6578,42135,1731507219475; forceNewPlan=false, retain=false 2024-11-13T14:13:42,799 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:42,800 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN; state=OFFLINE, location=eb1966fd6578,38145,1731507219374; forceNewPlan=false, retain=false 2024-11-13T14:13:42,802 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=cb7d9e65213673705aa2d05e9c49ef5f, ASSIGN 2024-11-13T14:13:42,803 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=cb7d9e65213673705aa2d05e9c49ef5f, ASSIGN; state=OFFLINE, location=eb1966fd6578,41771,1731507219528; forceNewPlan=false, retain=false 2024-11-13T14:13:42,805 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=409ba30f82abbec05bb6c84a023d1e9a, ASSIGN 2024-11-13T14:13:42,806 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=409ba30f82abbec05bb6c84a023d1e9a, ASSIGN; state=OFFLINE, location=eb1966fd6578,42135,1731507219475; forceNewPlan=false, retain=false 2024-11-13T14:13:42,808 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:42,809 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN; state=OFFLINE, location=eb1966fd6578,38145,1731507219374; forceNewPlan=false, retain=false 2024-11-13T14:13:42,811 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dbe405b0820b2bbf65d6f5d084e654ed, ASSIGN 2024-11-13T14:13:42,813 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dbe405b0820b2bbf65d6f5d084e654ed, ASSIGN; state=OFFLINE, location=eb1966fd6578,41771,1731507219528; forceNewPlan=false, retain=false 2024-11-13T14:13:42,816 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=668c3a8f00f6512df533713b2c2d46cb, ASSIGN 2024-11-13T14:13:42,818 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=668c3a8f00f6512df533713b2c2d46cb, ASSIGN; state=OFFLINE, location=eb1966fd6578,42135,1731507219475; forceNewPlan=false, retain=false 2024-11-13T14:13:42,820 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:42,821 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN; state=OFFLINE, location=eb1966fd6578,38145,1731507219374; forceNewPlan=false, retain=false 2024-11-13T14:13:42,823 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=17aeba8ae6d5f57b63d67825a02878d1, ASSIGN 2024-11-13T14:13:42,825 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=17aeba8ae6d5f57b63d67825a02878d1, ASSIGN; state=OFFLINE, location=eb1966fd6578,41771,1731507219528; forceNewPlan=false, retain=false 2024-11-13T14:13:42,828 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:42,830 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN; state=OFFLINE, location=eb1966fd6578,38145,1731507219374; forceNewPlan=false, retain=false 2024-11-13T14:13:42,832 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bc9bead55355bc7d07a869ba01fa7404, ASSIGN 2024-11-13T14:13:42,834 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bc9bead55355bc7d07a869ba01fa7404, ASSIGN; state=OFFLINE, location=eb1966fd6578,42135,1731507219475; forceNewPlan=false, retain=false 2024-11-13T14:13:42,836 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0e888576e09ecc86a3b2198a9e11c7b4, ASSIGN 2024-11-13T14:13:42,838 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0e888576e09ecc86a3b2198a9e11c7b4, ASSIGN; state=OFFLINE, location=eb1966fd6578,41771,1731507219528; forceNewPlan=false, retain=false 2024-11-13T14:13:42,839 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d2b53404c106b2f9960c2de982a64c06, ASSIGN 2024-11-13T14:13:42,840 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d2b53404c106b2f9960c2de982a64c06, ASSIGN; state=OFFLINE, location=eb1966fd6578,42135,1731507219475; forceNewPlan=false, retain=false 2024-11-13T14:13:42,842 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b2697dde157dfed90cc8595ff82dc081, ASSIGN 2024-11-13T14:13:42,844 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b2697dde157dfed90cc8595ff82dc081, ASSIGN; state=OFFLINE, location=eb1966fd6578,42135,1731507219475; forceNewPlan=false, retain=false 2024-11-13T14:13:42,919 INFO [eb1966fd6578:36813 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-13T14:13:42,921 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:42,923 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:42,924 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=1f231052a3b0401ad15e94c5bf55a972, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:42,926 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=026ea48c246e26b4197a66fa854f5022, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:42,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:42,928 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=4db6e838ae3d565bc2af54adcdf3a3c0, ASSIGN because future has completed 2024-11-13T14:13:42,928 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=668c3a8f00f6512df533713b2c2d46cb, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:42,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4db6e838ae3d565bc2af54adcdf3a3c0, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:42,931 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8ebd6e15e1eb3717f4dd8f1695d2b691, ASSIGN because future has completed 2024-11-13T14:13:42,933 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1f231052a3b0401ad15e94c5bf55a972, ASSIGN because future has completed 2024-11-13T14:13:42,936 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=026ea48c246e26b4197a66fa854f5022, ASSIGN because future has completed 2024-11-13T14:13:42,938 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=409ba30f82abbec05bb6c84a023d1e9a, ASSIGN because future has completed 2024-11-13T14:13:42,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 409ba30f82abbec05bb6c84a023d1e9a, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:42,940 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=668c3a8f00f6512df533713b2c2d46cb, ASSIGN because future has completed 2024-11-13T14:13:42,947 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 668c3a8f00f6512df533713b2c2d46cb, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:42,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 026ea48c246e26b4197a66fa854f5022, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:42,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f231052a3b0401ad15e94c5bf55a972, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:42,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ebd6e15e1eb3717f4dd8f1695d2b691, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:42,973 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bc9bead55355bc7d07a869ba01fa7404, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:42,974 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=d2b53404c106b2f9960c2de982a64c06, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:42,975 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b2697dde157dfed90cc8595ff82dc081, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:42,975 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9bab6e4180453738e691368cc5fbb429, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:42,976 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=24857182eb0da41497c7febd6e0c060d, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:42,977 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=edf3600642cf1af12a36dc726f301ef2, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:42,978 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e501f0ec953c86735bbe2081ac738865, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:42,978 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bc9bead55355bc7d07a869ba01fa7404, ASSIGN because future has completed 2024-11-13T14:13:42,979 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=087abf0ac7c707f0f540d20925d9d683, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:42,980 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:42,981 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:42,981 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:42,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:42,983 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:42,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bc9bead55355bc7d07a869ba01fa7404, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:42,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d2b53404c106b2f9960c2de982a64c06, ASSIGN because future has completed 2024-11-13T14:13:42,991 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b2697dde157dfed90cc8595ff82dc081, ASSIGN because future has completed 2024-11-13T14:13:42,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9bab6e4180453738e691368cc5fbb429, ASSIGN because future has completed 2024-11-13T14:13:42,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=24857182eb0da41497c7febd6e0c060d, ASSIGN because future has completed 2024-11-13T14:13:42,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24857182eb0da41497c7febd6e0c060d, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:42,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=edf3600642cf1af12a36dc726f301ef2, ASSIGN because future has completed 2024-11-13T14:13:42,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e501f0ec953c86735bbe2081ac738865, ASSIGN because future has completed 2024-11-13T14:13:43,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=087abf0ac7c707f0f540d20925d9d683, ASSIGN because future has completed 2024-11-13T14:13:43,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 087abf0ac7c707f0f540d20925d9d683, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:43,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=cb7d9e65213673705aa2d05e9c49ef5f, ASSIGN because future has completed 2024-11-13T14:13:43,005 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dbe405b0820b2bbf65d6f5d084e654ed, ASSIGN because future has completed 2024-11-13T14:13:43,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=17aeba8ae6d5f57b63d67825a02878d1, ASSIGN because future has completed 2024-11-13T14:13:43,009 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0e888576e09ecc86a3b2198a9e11c7b4, ASSIGN because future has completed 2024-11-13T14:13:43,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e888576e09ecc86a3b2198a9e11c7b4, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:43,011 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN because future has completed 2024-11-13T14:13:43,015 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,38145,1731507219374}] 2024-11-13T14:13:43,020 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17aeba8ae6d5f57b63d67825a02878d1, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:43,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbe405b0820b2bbf65d6f5d084e654ed, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:43,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb7d9e65213673705aa2d05e9c49ef5f, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:43,036 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure e501f0ec953c86735bbe2081ac738865, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:43,042 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure edf3600642cf1af12a36dc726f301ef2, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:43,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9bab6e4180453738e691368cc5fbb429, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:43,053 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b2697dde157dfed90cc8595ff82dc081, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:43,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b53404c106b2f9960c2de982a64c06, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:43,061 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,062 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,062 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,063 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,063 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,064 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=def71812430057c99b7be5281021b22c, regionState=OPENING, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN because future has completed 2024-11-13T14:13:43,065 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,38145,1731507219374}] 2024-11-13T14:13:43,066 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN because future has completed 2024-11-13T14:13:43,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN because future has completed 2024-11-13T14:13:43,068 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN because future has completed 2024-11-13T14:13:43,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN because future has completed 2024-11-13T14:13:43,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,38145,1731507219374}] 2024-11-13T14:13:43,073 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN because future has completed 2024-11-13T14:13:43,076 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure def71812430057c99b7be5281021b22c, server=eb1966fd6578,38145,1731507219374}] 2024-11-13T14:13:43,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbfee0cdbcdc3896eb3a9b618149bc3, server=eb1966fd6578,38145,1731507219374}] 2024-11-13T14:13:43,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,38145,1731507219374}] 2024-11-13T14:13:43,085 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T14:13:43,088 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37155, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T14:13:43,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,38145,1731507219374}] 2024-11-13T14:13:43,093 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,094 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:43,094 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => d2b53404c106b2f9960c2de982a64c06, NAME => 'test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-13T14:13:43,094 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:43,094 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,095 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:43,095 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:43,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN because future has completed 2024-11-13T14:13:43,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,38145,1731507219374}] 2024-11-13T14:13:43,097 INFO [StoreOpener-d2b53404c106b2f9960c2de982a64c06-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:43,099 INFO [StoreOpener-d2b53404c106b2f9960c2de982a64c06-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d2b53404c106b2f9960c2de982a64c06 columnFamilyName family 2024-11-13T14:13:43,099 DEBUG [StoreOpener-d2b53404c106b2f9960c2de982a64c06-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,100 INFO [StoreOpener-d2b53404c106b2f9960c2de982a64c06-1 {}] regionserver.HStore(327): Store=d2b53404c106b2f9960c2de982a64c06/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,100 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:43,102 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:43,102 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:43,103 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:43,103 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:43,105 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:43,108 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/d2b53404c106b2f9960c2de982a64c06/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,108 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened d2b53404c106b2f9960c2de982a64c06; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62561248, jitterRate=-0.0677647590637207}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,108 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:43,109 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for d2b53404c106b2f9960c2de982a64c06: Running coprocessor pre-open hook at 1731507223095Writing region info on filesystem at 1731507223095Initializing all the Stores at 1731507223096 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223097 (+1 ms)Cleaning up temporary data from old regions at 1731507223103 (+6 ms)Running coprocessor post-open hooks at 1731507223108 (+5 ms)Region opened successfully at 1731507223109 (+1 ms) 2024-11-13T14:13:43,111 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06., pid=49, masterSystemTime=1731507223085 2024-11-13T14:13:43,114 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:43,114 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:43,115 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:43,115 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 4db6e838ae3d565bc2af54adcdf3a3c0, NAME => 'test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-13T14:13:43,115 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:43,115 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,115 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:43,115 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:43,115 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:43,119 INFO [StoreOpener-4db6e838ae3d565bc2af54adcdf3a3c0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:43,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure d2b53404c106b2f9960c2de982a64c06, server=eb1966fd6578,42135,1731507219475 because future has completed 2024-11-13T14:13:43,121 INFO [StoreOpener-4db6e838ae3d565bc2af54adcdf3a3c0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4db6e838ae3d565bc2af54adcdf3a3c0 columnFamilyName family 2024-11-13T14:13:43,121 DEBUG [StoreOpener-4db6e838ae3d565bc2af54adcdf3a3c0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,122 INFO [StoreOpener-4db6e838ae3d565bc2af54adcdf3a3c0-1 {}] regionserver.HStore(327): Store=4db6e838ae3d565bc2af54adcdf3a3c0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,122 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:43,124 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:43,125 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:43,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=6 2024-11-13T14:13:43,125 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure d2b53404c106b2f9960c2de982a64c06, server=eb1966fd6578,42135,1731507219475 in 64 msec 2024-11-13T14:13:43,126 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:43,126 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:43,129 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:43,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d2b53404c106b2f9960c2de982a64c06, ASSIGN in 377 msec 2024-11-13T14:13:43,131 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/4db6e838ae3d565bc2af54adcdf3a3c0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,132 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 4db6e838ae3d565bc2af54adcdf3a3c0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61819776, jitterRate=-0.07881355285644531}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,132 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:43,132 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 4db6e838ae3d565bc2af54adcdf3a3c0: Running coprocessor pre-open hook at 1731507223116Writing region info on filesystem at 1731507223116Initializing all the Stores at 1731507223118 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223118Cleaning up temporary data from old regions at 1731507223126 (+8 ms)Running coprocessor post-open hooks at 1731507223132 (+6 ms)Region opened successfully at 1731507223132 2024-11-13T14:13:43,133 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0., pid=31, masterSystemTime=1731507223085 2024-11-13T14:13:43,135 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:43,135 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:43,136 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:43,136 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 668c3a8f00f6512df533713b2c2d46cb, NAME => 'test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-13T14:13:43,136 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:43,136 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:43,136 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,136 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:43,136 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:43,139 INFO [StoreOpener-668c3a8f00f6512df533713b2c2d46cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:43,139 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4db6e838ae3d565bc2af54adcdf3a3c0, server=eb1966fd6578,42135,1731507219475 because future has completed 2024-11-13T14:13:43,141 INFO [StoreOpener-668c3a8f00f6512df533713b2c2d46cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 668c3a8f00f6512df533713b2c2d46cb columnFamilyName family 2024-11-13T14:13:43,141 DEBUG [StoreOpener-668c3a8f00f6512df533713b2c2d46cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,142 INFO [StoreOpener-668c3a8f00f6512df533713b2c2d46cb-1 {}] regionserver.HStore(327): Store=668c3a8f00f6512df533713b2c2d46cb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,142 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:43,143 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:43,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-13T14:13:43,143 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 4db6e838ae3d565bc2af54adcdf3a3c0, server=eb1966fd6578,42135,1731507219475 in 211 msec 2024-11-13T14:13:43,143 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:43,144 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:43,144 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:43,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=4db6e838ae3d565bc2af54adcdf3a3c0, ASSIGN in 394 msec 2024-11-13T14:13:43,146 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:43,149 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T14:13:43,149 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/668c3a8f00f6512df533713b2c2d46cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,150 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44335, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T14:13:43,150 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 668c3a8f00f6512df533713b2c2d46cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71410223, jitterRate=0.06409524381160736}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,150 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:43,150 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 668c3a8f00f6512df533713b2c2d46cb: Running coprocessor pre-open hook at 1731507223137Writing region info on filesystem at 1731507223137Initializing all the Stores at 1731507223138 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223139 (+1 ms)Cleaning up temporary data from old regions at 1731507223144 (+5 ms)Running coprocessor post-open hooks at 1731507223150 (+6 ms)Region opened successfully at 1731507223150 2024-11-13T14:13:43,152 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb., pid=33, masterSystemTime=1731507223085 2024-11-13T14:13:43,155 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:43,156 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:43,156 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:43,156 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 8ebd6e15e1eb3717f4dd8f1695d2b691, NAME => 'test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-13T14:13:43,156 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:43,157 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:43,157 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,157 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:43,157 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:43,157 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:43,158 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 17aeba8ae6d5f57b63d67825a02878d1, NAME => 'test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-13T14:13:43,158 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:43,158 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,159 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:43,159 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:43,160 INFO [StoreOpener-8ebd6e15e1eb3717f4dd8f1695d2b691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:43,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 668c3a8f00f6512df533713b2c2d46cb, server=eb1966fd6578,42135,1731507219475 because future has completed 2024-11-13T14:13:43,161 INFO [StoreOpener-17aeba8ae6d5f57b63d67825a02878d1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:43,163 INFO [StoreOpener-8ebd6e15e1eb3717f4dd8f1695d2b691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8ebd6e15e1eb3717f4dd8f1695d2b691 columnFamilyName family 2024-11-13T14:13:43,163 INFO [StoreOpener-17aeba8ae6d5f57b63d67825a02878d1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 17aeba8ae6d5f57b63d67825a02878d1 columnFamilyName family 2024-11-13T14:13:43,163 DEBUG [StoreOpener-8ebd6e15e1eb3717f4dd8f1695d2b691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,163 DEBUG [StoreOpener-17aeba8ae6d5f57b63d67825a02878d1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,164 INFO [StoreOpener-17aeba8ae6d5f57b63d67825a02878d1-1 {}] regionserver.HStore(327): Store=17aeba8ae6d5f57b63d67825a02878d1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,164 INFO [StoreOpener-8ebd6e15e1eb3717f4dd8f1695d2b691-1 {}] regionserver.HStore(327): Store=8ebd6e15e1eb3717f4dd8f1695d2b691/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,164 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:43,164 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:43,166 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:43,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T14:13:43,167 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:43,167 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:43,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=12 2024-11-13T14:13:43,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 668c3a8f00f6512df533713b2c2d46cb, server=eb1966fd6578,42135,1731507219475 in 215 msec 2024-11-13T14:13:43,167 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:43,168 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:43,168 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:43,168 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:43,168 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:43,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=668c3a8f00f6512df533713b2c2d46cb, ASSIGN in 419 msec 2024-11-13T14:13:43,172 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:43,173 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:43,174 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:43,174 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => ee5b69e1e01798b1055c61a7885f87b4, NAME => 'test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-13T14:13:43,175 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:43,175 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,175 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:43,175 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:43,178 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/17aeba8ae6d5f57b63d67825a02878d1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,179 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/8ebd6e15e1eb3717f4dd8f1695d2b691/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,179 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 17aeba8ae6d5f57b63d67825a02878d1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69901536, jitterRate=0.04161405563354492}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,179 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:43,179 INFO [StoreOpener-ee5b69e1e01798b1055c61a7885f87b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:43,179 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 17aeba8ae6d5f57b63d67825a02878d1: Running coprocessor pre-open hook at 1731507223159Writing region info on filesystem at 1731507223159Initializing all the Stores at 1731507223160 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223160Cleaning up temporary data from old regions at 1731507223168 (+8 ms)Running coprocessor post-open hooks at 1731507223179 (+11 ms)Region opened successfully at 1731507223179 2024-11-13T14:13:43,180 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 8ebd6e15e1eb3717f4dd8f1695d2b691; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59917668, jitterRate=-0.10715717077255249}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,180 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:43,180 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 8ebd6e15e1eb3717f4dd8f1695d2b691: Running coprocessor pre-open hook at 1731507223157Writing region info on filesystem at 1731507223157Initializing all the Stores at 1731507223159 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223159Cleaning up temporary data from old regions at 1731507223168 (+9 ms)Running coprocessor post-open hooks at 1731507223180 (+12 ms)Region opened successfully at 1731507223180 2024-11-13T14:13:43,181 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1., pid=42, masterSystemTime=1731507223148 2024-11-13T14:13:43,181 INFO [StoreOpener-ee5b69e1e01798b1055c61a7885f87b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5b69e1e01798b1055c61a7885f87b4 columnFamilyName family 2024-11-13T14:13:43,182 DEBUG [StoreOpener-ee5b69e1e01798b1055c61a7885f87b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,182 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691., pid=36, masterSystemTime=1731507223085 2024-11-13T14:13:43,183 INFO [StoreOpener-ee5b69e1e01798b1055c61a7885f87b4-1 {}] regionserver.HStore(327): Store=ee5b69e1e01798b1055c61a7885f87b4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,183 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:43,184 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:43,185 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:43,186 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:43,185 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:43,186 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:43,186 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:43,186 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:43,186 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:43,186 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:43,186 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:43,186 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:43,186 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 24857182eb0da41497c7febd6e0c060d, NAME => 'test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-13T14:13:43,187 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:43,186 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => bc9bead55355bc7d07a869ba01fa7404, NAME => 'test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-13T14:13:43,187 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,187 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:43,187 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:43,188 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,187 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:43,188 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:43,188 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:43,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 17aeba8ae6d5f57b63d67825a02878d1, server=eb1966fd6578,41771,1731507219528 because future has completed 2024-11-13T14:13:43,190 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:43,190 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:43,190 INFO [StoreOpener-bc9bead55355bc7d07a869ba01fa7404-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:43,191 INFO [StoreOpener-24857182eb0da41497c7febd6e0c060d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:43,193 INFO [StoreOpener-bc9bead55355bc7d07a869ba01fa7404-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bc9bead55355bc7d07a869ba01fa7404 columnFamilyName family 2024-11-13T14:13:43,193 DEBUG [StoreOpener-bc9bead55355bc7d07a869ba01fa7404-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,194 INFO [StoreOpener-24857182eb0da41497c7febd6e0c060d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 24857182eb0da41497c7febd6e0c060d columnFamilyName family 2024-11-13T14:13:43,194 DEBUG [StoreOpener-24857182eb0da41497c7febd6e0c060d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,194 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8ebd6e15e1eb3717f4dd8f1695d2b691, server=eb1966fd6578,42135,1731507219475 because future has completed 2024-11-13T14:13:43,194 INFO [StoreOpener-bc9bead55355bc7d07a869ba01fa7404-1 {}] regionserver.HStore(327): Store=bc9bead55355bc7d07a869ba01fa7404/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,194 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:43,194 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ee5b69e1e01798b1055c61a7885f87b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,195 INFO [StoreOpener-24857182eb0da41497c7febd6e0c060d-1 {}] regionserver.HStore(327): Store=24857182eb0da41497c7febd6e0c060d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,195 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:43,195 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened ee5b69e1e01798b1055c61a7885f87b4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73725560, jitterRate=0.09859645366668701}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,195 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:43,195 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:43,195 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for ee5b69e1e01798b1055c61a7885f87b4: Running coprocessor pre-open hook at 1731507223176Writing region info on filesystem at 1731507223176Initializing all the Stores at 1731507223178 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223178Cleaning up temporary data from old regions at 1731507223186 (+8 ms)Running coprocessor post-open hooks at 1731507223195 (+9 ms)Region opened successfully at 1731507223195 2024-11-13T14:13:43,196 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:43,196 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:43,197 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4., pid=41, masterSystemTime=1731507223168 2024-11-13T14:13:43,197 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:43,198 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:43,198 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:43,199 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:43,199 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:43,200 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:43,201 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:43,201 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:43,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=25 2024-11-13T14:13:43,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 8ebd6e15e1eb3717f4dd8f1695d2b691, server=eb1966fd6578,42135,1731507219475 in 229 msec 2024-11-13T14:13:43,201 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d3783ebc90fc59392fa002c53f7c56f, NAME => 'test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-13T14:13:43,201 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:43,201 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,201 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:43,202 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:43,202 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:43,202 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:43,204 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8ebd6e15e1eb3717f4dd8f1695d2b691, ASSIGN in 452 msec 2024-11-13T14:13:43,205 INFO [StoreOpener-1d3783ebc90fc59392fa002c53f7c56f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:43,206 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/24857182eb0da41497c7febd6e0c060d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,206 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,206 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 24857182eb0da41497c7febd6e0c060d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67703520, jitterRate=0.008861064910888672}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,207 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:43,207 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 24857182eb0da41497c7febd6e0c060d: Running coprocessor pre-open hook at 1731507223188Writing region info on filesystem at 1731507223188Initializing all the Stores at 1731507223190 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223190Cleaning up temporary data from old regions at 1731507223199 (+9 ms)Running coprocessor post-open hooks at 1731507223207 (+8 ms)Region opened successfully at 1731507223207 2024-11-13T14:13:43,209 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d., pid=38, masterSystemTime=1731507223148 2024-11-13T14:13:43,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,38145,1731507219374 because future has completed 2024-11-13T14:13:43,210 INFO [StoreOpener-1d3783ebc90fc59392fa002c53f7c56f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d3783ebc90fc59392fa002c53f7c56f columnFamilyName family 2024-11-13T14:13:43,210 DEBUG [StoreOpener-1d3783ebc90fc59392fa002c53f7c56f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,211 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/bc9bead55355bc7d07a869ba01fa7404/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,211 INFO [StoreOpener-1d3783ebc90fc59392fa002c53f7c56f-1 {}] regionserver.HStore(327): Store=1d3783ebc90fc59392fa002c53f7c56f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,212 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:43,212 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened bc9bead55355bc7d07a869ba01fa7404; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59184335, jitterRate=-0.11808468401432037}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,212 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:43,212 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for bc9bead55355bc7d07a869ba01fa7404: Running coprocessor pre-open hook at 1731507223188Writing region info on filesystem at 1731507223188Initializing all the Stores at 1731507223190 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223190Cleaning up temporary data from old regions at 1731507223198 (+8 ms)Running coprocessor post-open hooks at 1731507223212 (+14 ms)Region opened successfully at 1731507223212 2024-11-13T14:13:43,213 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:43,213 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:43,213 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:43,213 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404., pid=37, masterSystemTime=1731507223085 2024-11-13T14:13:43,214 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:43,214 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => cb7d9e65213673705aa2d05e9c49ef5f, NAME => 'test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-13T14:13:43,214 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:43,214 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:43,214 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,214 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:43,214 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:43,215 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:43,215 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:43,217 INFO [StoreOpener-cb7d9e65213673705aa2d05e9c49ef5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:43,217 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:43,217 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:43,218 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:43,218 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 409ba30f82abbec05bb6c84a023d1e9a, NAME => 'test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-13T14:13:43,218 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:43,218 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,219 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:43,219 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:43,219 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:43,219 INFO [StoreOpener-cb7d9e65213673705aa2d05e9c49ef5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cb7d9e65213673705aa2d05e9c49ef5f columnFamilyName family 2024-11-13T14:13:43,219 DEBUG [StoreOpener-cb7d9e65213673705aa2d05e9c49ef5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=10 2024-11-13T14:13:43,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 17aeba8ae6d5f57b63d67825a02878d1, server=eb1966fd6578,41771,1731507219528 in 191 msec 2024-11-13T14:13:43,221 INFO [StoreOpener-cb7d9e65213673705aa2d05e9c49ef5f-1 {}] regionserver.HStore(327): Store=cb7d9e65213673705aa2d05e9c49ef5f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,221 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:43,221 INFO [StoreOpener-409ba30f82abbec05bb6c84a023d1e9a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:43,222 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:43,223 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:43,224 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=17aeba8ae6d5f57b63d67825a02878d1, ASSIGN in 471 msec 2024-11-13T14:13:43,224 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:43,224 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:43,224 INFO [StoreOpener-409ba30f82abbec05bb6c84a023d1e9a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 409ba30f82abbec05bb6c84a023d1e9a columnFamilyName family 2024-11-13T14:13:43,224 DEBUG [StoreOpener-409ba30f82abbec05bb6c84a023d1e9a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,226 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:43,226 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/1d3783ebc90fc59392fa002c53f7c56f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,226 INFO [StoreOpener-409ba30f82abbec05bb6c84a023d1e9a-1 {}] regionserver.HStore(327): Store=409ba30f82abbec05bb6c84a023d1e9a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,226 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:43,228 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 1d3783ebc90fc59392fa002c53f7c56f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68874109, jitterRate=0.026304200291633606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,228 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:43,228 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 1d3783ebc90fc59392fa002c53f7c56f: Running coprocessor pre-open hook at 1731507223202Writing region info on filesystem at 1731507223202Initializing all the Stores at 1731507223204 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223204Cleaning up temporary data from old regions at 1731507223216 (+12 ms)Running coprocessor post-open hooks at 1731507223228 (+12 ms)Region opened successfully at 1731507223228 2024-11-13T14:13:43,229 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:43,229 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:43,229 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f., pid=54, masterSystemTime=1731507223168 2024-11-13T14:13:43,231 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure bc9bead55355bc7d07a869ba01fa7404, server=eb1966fd6578,42135,1731507219475 because future has completed 2024-11-13T14:13:43,231 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:43,232 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:43,232 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:43,233 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:43,233 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,42135,1731507219475, table=test, region=bc9bead55355bc7d07a869ba01fa7404. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,234 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:43,234 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:43,234 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 2027122fd77a73d52ce4d3368dd23e0a, NAME => 'test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-13T14:13:43,234 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:43,235 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,235 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:43,235 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:43,237 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/cb7d9e65213673705aa2d05e9c49ef5f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,237 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:43,238 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened cb7d9e65213673705aa2d05e9c49ef5f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60672213, jitterRate=-0.09591357409954071}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,238 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:43,238 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for cb7d9e65213673705aa2d05e9c49ef5f: Running coprocessor pre-open hook at 1731507223215Writing region info on filesystem at 1731507223215Initializing all the Stores at 1731507223216 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223216Cleaning up temporary data from old regions at 1731507223224 (+8 ms)Running coprocessor post-open hooks at 1731507223238 (+14 ms)Region opened successfully at 1731507223238 2024-11-13T14:13:43,239 INFO [StoreOpener-2027122fd77a73d52ce4d3368dd23e0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:43,240 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f., pid=44, masterSystemTime=1731507223148 2024-11-13T14:13:43,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=8 2024-11-13T14:13:43,240 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure bc9bead55355bc7d07a869ba01fa7404, server=eb1966fd6578,42135,1731507219475 in 251 msec 2024-11-13T14:13:43,241 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,38145,1731507219374, table=test, region=ee5b69e1e01798b1055c61a7885f87b4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,243 INFO [StoreOpener-2027122fd77a73d52ce4d3368dd23e0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2027122fd77a73d52ce4d3368dd23e0a columnFamilyName family 2024-11-13T14:13:43,244 DEBUG [StoreOpener-2027122fd77a73d52ce4d3368dd23e0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,244 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/409ba30f82abbec05bb6c84a023d1e9a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,244 INFO [StoreOpener-2027122fd77a73d52ce4d3368dd23e0a-1 {}] regionserver.HStore(327): Store=2027122fd77a73d52ce4d3368dd23e0a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,245 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:43,245 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 409ba30f82abbec05bb6c84a023d1e9a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70605747, jitterRate=0.052107617259025574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,245 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:43,245 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 409ba30f82abbec05bb6c84a023d1e9a: Running coprocessor pre-open hook at 1731507223219Writing region info on filesystem at 1731507223219Initializing all the Stores at 1731507223220 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223220Cleaning up temporary data from old regions at 1731507223232 (+12 ms)Running coprocessor post-open hooks at 1731507223245 (+13 ms)Region opened successfully at 1731507223245 2024-11-13T14:13:43,246 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:43,246 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:43,247 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a., pid=32, masterSystemTime=1731507223085 2024-11-13T14:13:43,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bc9bead55355bc7d07a869ba01fa7404, ASSIGN in 491 msec 2024-11-13T14:13:43,247 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:43,247 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:43,248 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:43,248 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:43,249 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:43,249 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:43,249 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 087abf0ac7c707f0f540d20925d9d683, NAME => 'test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-13T14:13:43,250 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:43,250 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,250 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:43,250 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:43,251 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:43,251 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:43,252 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:43,252 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 026ea48c246e26b4197a66fa854f5022, NAME => 'test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-13T14:13:43,252 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:43,252 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure cb7d9e65213673705aa2d05e9c49ef5f, server=eb1966fd6578,41771,1731507219528 because future has completed 2024-11-13T14:13:43,252 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:43,253 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:43,253 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:43,253 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:43,257 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 409ba30f82abbec05bb6c84a023d1e9a, server=eb1966fd6578,42135,1731507219475 because future has completed 2024-11-13T14:13:43,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=16 2024-11-13T14:13:43,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure cb7d9e65213673705aa2d05e9c49ef5f, server=eb1966fd6578,41771,1731507219528 in 227 msec 2024-11-13T14:13:43,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=cb7d9e65213673705aa2d05e9c49ef5f, ASSIGN in 512 msec 2024-11-13T14:13:43,264 INFO [StoreOpener-087abf0ac7c707f0f540d20925d9d683-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:43,267 INFO [StoreOpener-087abf0ac7c707f0f540d20925d9d683-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 087abf0ac7c707f0f540d20925d9d683 columnFamilyName family 2024-11-13T14:13:43,267 DEBUG [StoreOpener-087abf0ac7c707f0f540d20925d9d683-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,268 INFO [StoreOpener-087abf0ac7c707f0f540d20925d9d683-1 {}] regionserver.HStore(327): Store=087abf0ac7c707f0f540d20925d9d683/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,268 INFO [StoreOpener-026ea48c246e26b4197a66fa854f5022-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:43,269 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:43,269 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/2027122fd77a73d52ce4d3368dd23e0a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=15 2024-11-13T14:13:43,270 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 409ba30f82abbec05bb6c84a023d1e9a, server=eb1966fd6578,42135,1731507219475 in 325 msec 2024-11-13T14:13:43,270 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 2027122fd77a73d52ce4d3368dd23e0a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69629939, jitterRate=0.03756694495677948}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,270 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:43,271 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 2027122fd77a73d52ce4d3368dd23e0a: Running coprocessor pre-open hook at 1731507223235Writing region info on filesystem at 1731507223235Initializing all the Stores at 1731507223237 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223237Cleaning up temporary data from old regions at 1731507223247 (+10 ms)Running coprocessor post-open hooks at 1731507223271 (+24 ms)Region opened successfully at 1731507223271 2024-11-13T14:13:43,272 INFO [StoreOpener-026ea48c246e26b4197a66fa854f5022-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 026ea48c246e26b4197a66fa854f5022 columnFamilyName family 2024-11-13T14:13:43,272 DEBUG [StoreOpener-026ea48c246e26b4197a66fa854f5022-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,273 INFO [StoreOpener-026ea48c246e26b4197a66fa854f5022-1 {}] regionserver.HStore(327): Store=026ea48c246e26b4197a66fa854f5022/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,273 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:43,274 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:43,275 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:43,275 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:43,275 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:43,278 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:43,278 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:43,279 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:43,279 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a., pid=50, masterSystemTime=1731507223168 2024-11-13T14:13:43,280 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:43,280 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:43,282 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:43,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=409ba30f82abbec05bb6c84a023d1e9a, ASSIGN in 521 msec 2024-11-13T14:13:43,283 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:43,283 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:43,283 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => ef7ba06bec150589d1c840bbd1b13d9b, NAME => 'test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-13T14:13:43,283 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:43,284 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,284 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:43,284 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:43,285 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/026ea48c246e26b4197a66fa854f5022/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,286 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,287 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:43,287 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 026ea48c246e26b4197a66fa854f5022; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67129439, jitterRate=3.0659139156341553E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,287 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:43,287 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 026ea48c246e26b4197a66fa854f5022: Running coprocessor pre-open hook at 1731507223253Writing region info on filesystem at 1731507223253Initializing all the Stores at 1731507223254 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223254Cleaning up temporary data from old regions at 1731507223275 (+21 ms)Running coprocessor post-open hooks at 1731507223287 (+12 ms)Region opened successfully at 1731507223287 2024-11-13T14:13:43,288 INFO [StoreOpener-ef7ba06bec150589d1c840bbd1b13d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:43,290 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022., pid=34, masterSystemTime=1731507223085 2024-11-13T14:13:43,291 INFO [StoreOpener-ef7ba06bec150589d1c840bbd1b13d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef7ba06bec150589d1c840bbd1b13d9b columnFamilyName family 2024-11-13T14:13:43,292 DEBUG [StoreOpener-ef7ba06bec150589d1c840bbd1b13d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,293 INFO [StoreOpener-ef7ba06bec150589d1c840bbd1b13d9b-1 {}] regionserver.HStore(327): Store=ef7ba06bec150589d1c840bbd1b13d9b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,293 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,38145,1731507219374 because future has completed 2024-11-13T14:13:43,293 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:43,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,294 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:43,294 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:43,294 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:43,294 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f231052a3b0401ad15e94c5bf55a972, NAME => 'test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-13T14:13:43,294 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:43,295 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,295 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:43,295 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:43,295 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:43,297 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:43,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,38145,1731507219374 because future has completed 2024-11-13T14:13:43,298 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:43,299 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:43,299 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:43,301 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/087abf0ac7c707f0f540d20925d9d683/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,303 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 087abf0ac7c707f0f540d20925d9d683; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60395963, jitterRate=-0.10003001987934113}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,303 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:43,303 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 087abf0ac7c707f0f540d20925d9d683: Running coprocessor pre-open hook at 1731507223250Writing region info on filesystem at 1731507223250Initializing all the Stores at 1731507223254 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223254Cleaning up temporary data from old regions at 1731507223280 (+26 ms)Running coprocessor post-open hooks at 1731507223303 (+23 ms)Region opened successfully at 1731507223303 2024-11-13T14:13:43,305 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:43,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 026ea48c246e26b4197a66fa854f5022, server=eb1966fd6578,42135,1731507219475 because future has completed 2024-11-13T14:13:43,305 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683., pid=39, masterSystemTime=1731507223148 2024-11-13T14:13:43,309 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:43,309 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:43,309 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:43,310 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => e501f0ec953c86735bbe2081ac738865, NAME => 'test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-13T14:13:43,310 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:43,310 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,310 INFO [StoreOpener-1f231052a3b0401ad15e94c5bf55a972-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:43,310 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:43,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=20 2024-11-13T14:13:43,310 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:43,310 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,38145,1731507219374 in 222 msec 2024-11-13T14:13:43,311 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ef7ba06bec150589d1c840bbd1b13d9b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,312 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened ef7ba06bec150589d1c840bbd1b13d9b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69723148, jitterRate=0.038955867290496826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,312 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:43,312 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for ef7ba06bec150589d1c840bbd1b13d9b: Running coprocessor pre-open hook at 1731507223284Writing region info on filesystem at 1731507223284Initializing all the Stores at 1731507223286 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223286Cleaning up temporary data from old regions at 1731507223299 (+13 ms)Running coprocessor post-open hooks at 1731507223312 (+13 ms)Region opened successfully at 1731507223312 2024-11-13T14:13:43,313 INFO [StoreOpener-1f231052a3b0401ad15e94c5bf55a972-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f231052a3b0401ad15e94c5bf55a972 columnFamilyName family 2024-11-13T14:13:43,313 INFO [StoreOpener-e501f0ec953c86735bbe2081ac738865-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:43,313 DEBUG [StoreOpener-1f231052a3b0401ad15e94c5bf55a972-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,314 INFO [StoreOpener-1f231052a3b0401ad15e94c5bf55a972-1 {}] regionserver.HStore(327): Store=1f231052a3b0401ad15e94c5bf55a972/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,314 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b., pid=51, masterSystemTime=1731507223168 2024-11-13T14:13:43,314 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:43,315 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:43,316 INFO [StoreOpener-e501f0ec953c86735bbe2081ac738865-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e501f0ec953c86735bbe2081ac738865 columnFamilyName family 2024-11-13T14:13:43,316 DEBUG [StoreOpener-e501f0ec953c86735bbe2081ac738865-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN in 562 msec 2024-11-13T14:13:43,316 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:43,317 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:43,317 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:43,317 INFO [StoreOpener-e501f0ec953c86735bbe2081ac738865-1 {}] regionserver.HStore(327): Store=e501f0ec953c86735bbe2081ac738865/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,317 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:43,317 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:43,317 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:43,317 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:43,317 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => def71812430057c99b7be5281021b22c, NAME => 'test,ggg,1731507221993.def71812430057c99b7be5281021b22c.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-13T14:13:43,317 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test def71812430057c99b7be5281021b22c 2024-11-13T14:13:43,318 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ggg,1731507221993.def71812430057c99b7be5281021b22c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,318 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for def71812430057c99b7be5281021b22c 2024-11-13T14:13:43,318 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for def71812430057c99b7be5281021b22c 2024-11-13T14:13:43,318 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:43,320 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,320 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:43,321 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:43,321 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:43,321 INFO [StoreOpener-def71812430057c99b7be5281021b22c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region def71812430057c99b7be5281021b22c 2024-11-13T14:13:43,321 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:43,324 INFO [StoreOpener-def71812430057c99b7be5281021b22c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region def71812430057c99b7be5281021b22c columnFamilyName family 2024-11-13T14:13:43,325 DEBUG [StoreOpener-def71812430057c99b7be5281021b22c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,326 INFO [StoreOpener-def71812430057c99b7be5281021b22c-1 {}] regionserver.HStore(327): Store=def71812430057c99b7be5281021b22c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,326 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for def71812430057c99b7be5281021b22c 2024-11-13T14:13:43,327 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:43,328 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,38145,1731507219374 because future has completed 2024-11-13T14:13:43,328 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/def71812430057c99b7be5281021b22c 2024-11-13T14:13:43,328 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:43,329 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/def71812430057c99b7be5281021b22c 2024-11-13T14:13:43,329 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for def71812430057c99b7be5281021b22c 2024-11-13T14:13:43,330 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for def71812430057c99b7be5281021b22c 2024-11-13T14:13:43,333 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,41771,1731507219528, table=test, region=087abf0ac7c707f0f540d20925d9d683. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 087abf0ac7c707f0f540d20925d9d683, server=eb1966fd6578,41771,1731507219528 because future has completed 2024-11-13T14:13:43,336 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for def71812430057c99b7be5281021b22c 2024-11-13T14:13:43,338 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,42135,1731507219475, table=test, region=026ea48c246e26b4197a66fa854f5022. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,340 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/1f231052a3b0401ad15e94c5bf55a972/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=14 2024-11-13T14:13:43,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,38145,1731507219374 in 262 msec 2024-11-13T14:13:43,341 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 1f231052a3b0401ad15e94c5bf55a972; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73776342, jitterRate=0.09935316443443298}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,341 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:43,342 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 1f231052a3b0401ad15e94c5bf55a972: Running coprocessor pre-open hook at 1731507223295Writing region info on filesystem at 1731507223295Initializing all the Stores at 1731507223298 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223298Cleaning up temporary data from old regions at 1731507223317 (+19 ms)Running coprocessor post-open hooks at 1731507223341 (+24 ms)Region opened successfully at 1731507223342 (+1 ms) 2024-11-13T14:13:43,343 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972., pid=35, masterSystemTime=1731507223085 2024-11-13T14:13:43,346 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,38145,1731507219374, table=test, region=2027122fd77a73d52ce4d3368dd23e0a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,346 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,38145,1731507219374, table=test, region=ee5b69e1e01798b1055c61a7885f87b4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,347 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:43,347 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:43,347 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:43,347 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => b2697dde157dfed90cc8595ff82dc081, NAME => 'test,,1731507221993.b2697dde157dfed90cc8595ff82dc081.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-13T14:13:43,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN in 592 msec 2024-11-13T14:13:43,348 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:43,348 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,,1731507221993.b2697dde157dfed90cc8595ff82dc081.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,348 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:43,348 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:43,350 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/e501f0ec953c86735bbe2081ac738865/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,350 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:43,351 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/def71812430057c99b7be5281021b22c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,351 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened e501f0ec953c86735bbe2081ac738865; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60509081, jitterRate=-0.09834443032741547}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,351 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:43,351 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for e501f0ec953c86735bbe2081ac738865: Running coprocessor pre-open hook at 1731507223310Writing region info on filesystem at 1731507223310Initializing all the Stores at 1731507223312 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223312Cleaning up temporary data from old regions at 1731507223321 (+9 ms)Running coprocessor post-open hooks at 1731507223351 (+30 ms)Region opened successfully at 1731507223351 2024-11-13T14:13:43,352 INFO [StoreOpener-b2697dde157dfed90cc8595ff82dc081-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:43,352 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened def71812430057c99b7be5281021b22c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62473817, jitterRate=-0.06906758248806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,353 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for def71812430057c99b7be5281021b22c 2024-11-13T14:13:43,353 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865., pid=45, masterSystemTime=1731507223148 2024-11-13T14:13:43,353 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for def71812430057c99b7be5281021b22c: Running coprocessor pre-open hook at 1731507223318Writing region info on filesystem at 1731507223318Initializing all the Stores at 1731507223320 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223320Cleaning up temporary data from old regions at 1731507223330 (+10 ms)Running coprocessor post-open hooks at 1731507223353 (+23 ms)Region opened successfully at 1731507223353 2024-11-13T14:13:43,354 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731507221993.def71812430057c99b7be5281021b22c., pid=52, masterSystemTime=1731507223168 2024-11-13T14:13:43,355 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1f231052a3b0401ad15e94c5bf55a972, server=eb1966fd6578,42135,1731507219475 because future has completed 2024-11-13T14:13:43,357 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:43,357 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:43,357 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:43,358 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 9bab6e4180453738e691368cc5fbb429, NAME => 'test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-13T14:13:43,358 INFO [StoreOpener-b2697dde157dfed90cc8595ff82dc081-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b2697dde157dfed90cc8595ff82dc081 columnFamilyName family 2024-11-13T14:13:43,358 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:43,358 DEBUG [StoreOpener-b2697dde157dfed90cc8595ff82dc081-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,358 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,358 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:43,358 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:43,359 INFO [StoreOpener-b2697dde157dfed90cc8595ff82dc081-1 {}] regionserver.HStore(327): Store=b2697dde157dfed90cc8595ff82dc081/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,359 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:43,361 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:43,361 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:43,361 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:43,362 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:43,362 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:43,362 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:43,362 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:43,363 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => edbfee0cdbcdc3896eb3a9b618149bc3, NAME => 'test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-13T14:13:43,364 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:43,364 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,365 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:43,365 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:43,366 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:43,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=19 2024-11-13T14:13:43,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 087abf0ac7c707f0f540d20925d9d683, server=eb1966fd6578,41771,1731507219528 in 358 msec 2024-11-13T14:13:43,369 INFO [StoreOpener-9bab6e4180453738e691368cc5fbb429-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:43,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=087abf0ac7c707f0f540d20925d9d683, ASSIGN in 618 msec 2024-11-13T14:13:43,371 INFO [StoreOpener-9bab6e4180453738e691368cc5fbb429-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9bab6e4180453738e691368cc5fbb429 columnFamilyName family 2024-11-13T14:13:43,372 DEBUG [StoreOpener-9bab6e4180453738e691368cc5fbb429-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,372 INFO [StoreOpener-9bab6e4180453738e691368cc5fbb429-1 {}] regionserver.HStore(327): Store=9bab6e4180453738e691368cc5fbb429/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,373 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:43,373 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=def71812430057c99b7be5281021b22c, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,374 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:43,375 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:43,376 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:43,376 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:43,377 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure def71812430057c99b7be5281021b22c, server=eb1966fd6578,38145,1731507219374 because future has completed 2024-11-13T14:13:43,377 INFO [StoreOpener-edbfee0cdbcdc3896eb3a9b618149bc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:43,378 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:43,379 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:43,381 INFO [StoreOpener-edbfee0cdbcdc3896eb3a9b618149bc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edbfee0cdbcdc3896eb3a9b618149bc3 columnFamilyName family 2024-11-13T14:13:43,381 DEBUG [StoreOpener-edbfee0cdbcdc3896eb3a9b618149bc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure e501f0ec953c86735bbe2081ac738865, server=eb1966fd6578,41771,1731507219528 because future has completed 2024-11-13T14:13:43,382 INFO [StoreOpener-edbfee0cdbcdc3896eb3a9b618149bc3-1 {}] regionserver.HStore(327): Store=edbfee0cdbcdc3896eb3a9b618149bc3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,383 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:43,384 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:43,385 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:43,385 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/b2697dde157dfed90cc8595ff82dc081/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,385 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:43,385 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:43,386 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened b2697dde157dfed90cc8595ff82dc081; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72272413, jitterRate=0.07694287598133087}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,386 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:43,386 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for b2697dde157dfed90cc8595ff82dc081: Running coprocessor pre-open hook at 1731507223348Writing region info on filesystem at 1731507223348Initializing all the Stores at 1731507223350 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223350Cleaning up temporary data from old regions at 1731507223362 (+12 ms)Running coprocessor post-open hooks at 1731507223386 (+24 ms)Region opened successfully at 1731507223386 2024-11-13T14:13:43,387 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731507221993.b2697dde157dfed90cc8595ff82dc081., pid=48, masterSystemTime=1731507223085 2024-11-13T14:13:43,388 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:43,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=11 2024-11-13T14:13:43,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure def71812430057c99b7be5281021b22c, server=eb1966fd6578,38145,1731507219374 in 308 msec 2024-11-13T14:13:43,392 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:43,392 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:43,393 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/9bab6e4180453738e691368cc5fbb429/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,394 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 9bab6e4180453738e691368cc5fbb429; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71724646, jitterRate=0.06878051161766052}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,395 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:43,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN in 643 msec 2024-11-13T14:13:43,395 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 9bab6e4180453738e691368cc5fbb429: Running coprocessor pre-open hook at 1731507223359Writing region info on filesystem at 1731507223359Initializing all the Stores at 1731507223360 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223361 (+1 ms)Cleaning up temporary data from old regions at 1731507223376 (+15 ms)Running coprocessor post-open hooks at 1731507223395 (+19 ms)Region opened successfully at 1731507223395 2024-11-13T14:13:43,396 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429., pid=47, masterSystemTime=1731507223148 2024-11-13T14:13:43,396 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:43,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure b2697dde157dfed90cc8595ff82dc081, server=eb1966fd6578,42135,1731507219475 because future has completed 2024-11-13T14:13:43,401 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/edbfee0cdbcdc3896eb3a9b618149bc3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,402 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened edbfee0cdbcdc3896eb3a9b618149bc3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63435032, jitterRate=-0.05474436283111572}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,402 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:43,402 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for edbfee0cdbcdc3896eb3a9b618149bc3: Running coprocessor pre-open hook at 1731507223365Writing region info on filesystem at 1731507223365Initializing all the Stores at 1731507223367 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223367Cleaning up temporary data from old regions at 1731507223385 (+18 ms)Running coprocessor post-open hooks at 1731507223402 (+17 ms)Region opened successfully at 1731507223402 2024-11-13T14:13:43,403 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3., pid=53, masterSystemTime=1731507223168 2024-11-13T14:13:43,409 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:43,409 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:43,409 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:43,410 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 0e888576e09ecc86a3b2198a9e11c7b4, NAME => 'test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-13T14:13:43,410 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:43,410 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,410 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:43,410 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:43,411 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:43,411 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:43,411 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:43,411 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => eb1bb70b6e4e99eeece01c54dd8a665c, NAME => 'test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-13T14:13:43,412 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:43,412 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,412 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:43,412 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:43,413 INFO [StoreOpener-0e888576e09ecc86a3b2198a9e11c7b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:43,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=5 2024-11-13T14:13:43,414 INFO [StoreOpener-eb1bb70b6e4e99eeece01c54dd8a665c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:43,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure b2697dde157dfed90cc8595ff82dc081, server=eb1966fd6578,42135,1731507219475 in 357 msec 2024-11-13T14:13:43,416 INFO [StoreOpener-0e888576e09ecc86a3b2198a9e11c7b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0e888576e09ecc86a3b2198a9e11c7b4 columnFamilyName family 2024-11-13T14:13:43,416 DEBUG [StoreOpener-0e888576e09ecc86a3b2198a9e11c7b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,416 INFO [StoreOpener-0e888576e09ecc86a3b2198a9e11c7b4-1 {}] regionserver.HStore(327): Store=0e888576e09ecc86a3b2198a9e11c7b4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,417 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:43,417 INFO [StoreOpener-eb1bb70b6e4e99eeece01c54dd8a665c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb1bb70b6e4e99eeece01c54dd8a665c columnFamilyName family 2024-11-13T14:13:43,417 DEBUG [StoreOpener-eb1bb70b6e4e99eeece01c54dd8a665c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b2697dde157dfed90cc8595ff82dc081, ASSIGN in 665 msec 2024-11-13T14:13:43,418 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:43,418 INFO [StoreOpener-eb1bb70b6e4e99eeece01c54dd8a665c-1 {}] regionserver.HStore(327): Store=eb1bb70b6e4e99eeece01c54dd8a665c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,418 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:43,419 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:43,419 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,419 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:43,419 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:43,420 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:43,422 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:43,422 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:43,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbfee0cdbcdc3896eb3a9b618149bc3, server=eb1966fd6578,38145,1731507219374 because future has completed 2024-11-13T14:13:43,423 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:43,423 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:43,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:43,425 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:43,427 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/0e888576e09ecc86a3b2198a9e11c7b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,428 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 0e888576e09ecc86a3b2198a9e11c7b4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75010954, jitterRate=0.11775031685829163}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,428 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:43,428 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 0e888576e09ecc86a3b2198a9e11c7b4: Running coprocessor pre-open hook at 1731507223411Writing region info on filesystem at 1731507223411Initializing all the Stores at 1731507223412 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223412Cleaning up temporary data from old regions at 1731507223419 (+7 ms)Running coprocessor post-open hooks at 1731507223428 (+9 ms)Region opened successfully at 1731507223428 2024-11-13T14:13:43,429 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4., pid=40, masterSystemTime=1731507223148 2024-11-13T14:13:43,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9bab6e4180453738e691368cc5fbb429, server=eb1966fd6578,41771,1731507219528 because future has completed 2024-11-13T14:13:43,431 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/eb1bb70b6e4e99eeece01c54dd8a665c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,432 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened eb1bb70b6e4e99eeece01c54dd8a665c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60271485, jitterRate=-0.1018848866224289}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,432 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:43,432 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for eb1bb70b6e4e99eeece01c54dd8a665c: Running coprocessor pre-open hook at 1731507223412Writing region info on filesystem at 1731507223412Initializing all the Stores at 1731507223414 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223414Cleaning up temporary data from old regions at 1731507223423 (+9 ms)Running coprocessor post-open hooks at 1731507223432 (+9 ms)Region opened successfully at 1731507223432 2024-11-13T14:13:43,433 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:43,433 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:43,433 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:43,433 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c., pid=56, masterSystemTime=1731507223168 2024-11-13T14:13:43,433 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => dbe405b0820b2bbf65d6f5d084e654ed, NAME => 'test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-13T14:13:43,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=17 2024-11-13T14:13:43,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure edbfee0cdbcdc3896eb3a9b618149bc3, server=eb1966fd6578,38145,1731507219374 in 350 msec 2024-11-13T14:13:43,435 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:43,435 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,435 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:43,436 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:43,436 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:43,436 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:43,437 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:43,437 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => dd3915f2bd1ed9356a65d71ba154c88b, NAME => 'test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-13T14:13:43,437 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:43,437 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,437 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:43,438 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:43,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN in 686 msec 2024-11-13T14:13:43,438 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,41771,1731507219528, table=test, region=e501f0ec953c86735bbe2081ac738865. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,438 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,41771,1731507219528, table=test, region=9bab6e4180453738e691368cc5fbb429. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,439 INFO [StoreOpener-dbe405b0820b2bbf65d6f5d084e654ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:43,440 INFO [StoreOpener-dd3915f2bd1ed9356a65d71ba154c88b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:43,441 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,443 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,42135,1731507219475, table=test, region=026ea48c246e26b4197a66fa854f5022. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,444 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,42135,1731507219475, table=test, region=1f231052a3b0401ad15e94c5bf55a972. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,444 INFO [StoreOpener-dbe405b0820b2bbf65d6f5d084e654ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dbe405b0820b2bbf65d6f5d084e654ed columnFamilyName family 2024-11-13T14:13:43,444 INFO [StoreOpener-dd3915f2bd1ed9356a65d71ba154c88b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd3915f2bd1ed9356a65d71ba154c88b columnFamilyName family 2024-11-13T14:13:43,444 DEBUG [StoreOpener-dbe405b0820b2bbf65d6f5d084e654ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,444 DEBUG [StoreOpener-dd3915f2bd1ed9356a65d71ba154c88b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,445 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,38145,1731507219374 because future has completed 2024-11-13T14:13:43,445 INFO [StoreOpener-dd3915f2bd1ed9356a65d71ba154c88b-1 {}] regionserver.HStore(327): Store=dd3915f2bd1ed9356a65d71ba154c88b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,446 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:43,446 INFO [StoreOpener-dbe405b0820b2bbf65d6f5d084e654ed-1 {}] regionserver.HStore(327): Store=dbe405b0820b2bbf65d6f5d084e654ed/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,446 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:43,447 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:43,447 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:43,447 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:43,447 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:43,448 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:43,448 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:43,451 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:43,452 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0e888576e09ecc86a3b2198a9e11c7b4, server=eb1966fd6578,41771,1731507219528 because future has completed 2024-11-13T14:13:43,452 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:43,452 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:43,452 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:43,454 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,38145,1731507219374, table=test, region=2027122fd77a73d52ce4d3368dd23e0a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,454 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,38145,1731507219374, table=test, region=ee5b69e1e01798b1055c61a7885f87b4. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,457 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:43,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=9 2024-11-13T14:13:43,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,38145,1731507219374 in 357 msec 2024-11-13T14:13:43,458 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/dd3915f2bd1ed9356a65d71ba154c88b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,459 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened dd3915f2bd1ed9356a65d71ba154c88b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66065144, jitterRate=-0.015552639961242676}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,459 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:43,459 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for dd3915f2bd1ed9356a65d71ba154c88b: Running coprocessor pre-open hook at 1731507223438Writing region info on filesystem at 1731507223438Initializing all the Stores at 1731507223439 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223439Cleaning up temporary data from old regions at 1731507223448 (+9 ms)Running coprocessor post-open hooks at 1731507223459 (+11 ms)Region opened successfully at 1731507223459 2024-11-13T14:13:43,460 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b., pid=55, masterSystemTime=1731507223168 2024-11-13T14:13:43,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN in 709 msec 2024-11-13T14:13:43,463 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/dbe405b0820b2bbf65d6f5d084e654ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,464 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:43,464 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:43,464 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened dbe405b0820b2bbf65d6f5d084e654ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67179172, jitterRate=0.001047670841217041}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,465 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:43,465 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for dbe405b0820b2bbf65d6f5d084e654ed: Running coprocessor pre-open hook at 1731507223436Writing region info on filesystem at 1731507223436Initializing all the Stores at 1731507223438 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223438Cleaning up temporary data from old regions at 1731507223452 (+14 ms)Running coprocessor post-open hooks at 1731507223465 (+13 ms)Region opened successfully at 1731507223465 2024-11-13T14:13:43,466 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed., pid=43, masterSystemTime=1731507223148 2024-11-13T14:13:43,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=7 2024-11-13T14:13:43,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 0e888576e09ecc86a3b2198a9e11c7b4, server=eb1966fd6578,41771,1731507219528 in 453 msec 2024-11-13T14:13:43,468 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:43,468 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:43,469 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:43,469 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => edf3600642cf1af12a36dc726f301ef2, NAME => 'test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-13T14:13:43,469 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:43,469 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:43,469 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:43,469 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:43,471 INFO [StoreOpener-edf3600642cf1af12a36dc726f301ef2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:43,472 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0e888576e09ecc86a3b2198a9e11c7b4, ASSIGN in 719 msec 2024-11-13T14:13:43,474 INFO [StoreOpener-edf3600642cf1af12a36dc726f301ef2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edf3600642cf1af12a36dc726f301ef2 columnFamilyName family 2024-11-13T14:13:43,474 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:43,474 DEBUG [StoreOpener-edf3600642cf1af12a36dc726f301ef2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:43,475 INFO [StoreOpener-edf3600642cf1af12a36dc726f301ef2-1 {}] regionserver.HStore(327): Store=edf3600642cf1af12a36dc726f301ef2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:43,475 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:43,476 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:43,477 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:43,477 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure dbe405b0820b2bbf65d6f5d084e654ed, server=eb1966fd6578,41771,1731507219528 because future has completed 2024-11-13T14:13:43,477 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:43,477 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:43,478 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,38145,1731507219374 2024-11-13T14:13:43,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,38145,1731507219374 because future has completed 2024-11-13T14:13:43,481 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:43,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=13 2024-11-13T14:13:43,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure dbe405b0820b2bbf65d6f5d084e654ed, server=eb1966fd6578,41771,1731507219528 in 455 msec 2024-11-13T14:13:43,486 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/edf3600642cf1af12a36dc726f301ef2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T14:13:43,486 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened edf3600642cf1af12a36dc726f301ef2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61952901, jitterRate=-0.07682983577251434}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:43,487 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:43,487 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for edf3600642cf1af12a36dc726f301ef2: Running coprocessor pre-open hook at 1731507223470Writing region info on filesystem at 1731507223470Initializing all the Stores at 1731507223471 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507223471Cleaning up temporary data from old regions at 1731507223477 (+6 ms)Running coprocessor post-open hooks at 1731507223487 (+10 ms)Region opened successfully at 1731507223487 2024-11-13T14:13:43,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dbe405b0820b2bbf65d6f5d084e654ed, ASSIGN in 736 msec 2024-11-13T14:13:43,488 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2., pid=46, masterSystemTime=1731507223148 2024-11-13T14:13:43,490 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:43,490 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:43,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=24 2024-11-13T14:13:43,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,38145,1731507219374 in 401 msec 2024-11-13T14:13:43,494 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN in 743 msec 2024-11-13T14:13:43,495 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:43,498 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure edf3600642cf1af12a36dc726f301ef2, server=eb1966fd6578,41771,1731507219528 because future has completed 2024-11-13T14:13:43,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=29 2024-11-13T14:13:43,503 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 9bab6e4180453738e691368cc5fbb429, server=eb1966fd6578,41771,1731507219528 in 453 msec 2024-11-13T14:13:43,505 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9bab6e4180453738e691368cc5fbb429, ASSIGN in 754 msec 2024-11-13T14:13:43,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=23 2024-11-13T14:13:43,510 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure edf3600642cf1af12a36dc726f301ef2, server=eb1966fd6578,41771,1731507219528 in 465 msec 2024-11-13T14:13:43,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=edf3600642cf1af12a36dc726f301ef2, ASSIGN in 761 msec 2024-11-13T14:13:43,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=22 2024-11-13T14:13:43,517 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure e501f0ec953c86735bbe2081ac738865, server=eb1966fd6578,41771,1731507219528 in 478 msec 2024-11-13T14:13:43,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e501f0ec953c86735bbe2081ac738865, ASSIGN in 768 msec 2024-11-13T14:13:43,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=21 2024-11-13T14:13:43,523 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 1f231052a3b0401ad15e94c5bf55a972, server=eb1966fd6578,42135,1731507219475 in 560 msec 2024-11-13T14:13:43,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1f231052a3b0401ad15e94c5bf55a972, ASSIGN in 774 msec 2024-11-13T14:13:43,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=18 2024-11-13T14:13:43,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 026ea48c246e26b4197a66fa854f5022, server=eb1966fd6578,42135,1731507219475 in 572 msec 2024-11-13T14:13:43,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=026ea48c246e26b4197a66fa854f5022, ASSIGN in 780 msec 2024-11-13T14:13:43,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=27 2024-11-13T14:13:43,536 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,38145,1731507219374 in 468 msec 2024-11-13T14:13:43,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN in 787 msec 2024-11-13T14:13:43,539 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, openSeqNum=2, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:43,542 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,41771,1731507219528, table=test, region=24857182eb0da41497c7febd6e0c060d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:43,543 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure 24857182eb0da41497c7febd6e0c060d, server=eb1966fd6578,41771,1731507219528 because future has completed 2024-11-13T14:13:43,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=30 2024-11-13T14:13:43,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,38145,1731507219374 in 528 msec 2024-11-13T14:13:43,549 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN in 797 msec 2024-11-13T14:13:43,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=26 2024-11-13T14:13:43,555 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure 24857182eb0da41497c7febd6e0c060d, server=eb1966fd6578,41771,1731507219528 in 556 msec 2024-11-13T14:13:43,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=4 2024-11-13T14:13:43,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=24857182eb0da41497c7febd6e0c060d, ASSIGN in 806 msec 2024-11-13T14:13:43,561 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T14:13:43,561 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731507223561"}]},"ts":"1731507223561"} 2024-11-13T14:13:43,564 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-13T14:13:43,566 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T14:13:43,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5630 sec 2024-11-13T14:13:44,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T14:13:44,179 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-13T14:13:44,179 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-13T14:13:44,182 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T14:13:44,200 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-13T14:13:44,200 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T14:13:44,201 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-13T14:13:44,201 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T14:13:44,210 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-13T14:13:44,234 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-13T14:13:44,249 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T14:13:44,251 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T14:13:44,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T14:13:44,258 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50510, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T14:13:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region eb1bb70b6e4e99eeece01c54dd8a665c changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region def71812430057c99b7be5281021b22c changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ef7ba06bec150589d1c840bbd1b13d9b changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region edbfee0cdbcdc3896eb3a9b618149bc3 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1d3783ebc90fc59392fa002c53f7c56f changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dd3915f2bd1ed9356a65d71ba154c88b changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2027122fd77a73d52ce4d3368dd23e0a changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ee5b69e1e01798b1055c61a7885f87b4 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0e888576e09ecc86a3b2198a9e11c7b4 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 17aeba8ae6d5f57b63d67825a02878d1 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dbe405b0820b2bbf65d6f5d084e654ed changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region cb7d9e65213673705aa2d05e9c49ef5f changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 087abf0ac7c707f0f540d20925d9d683 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e501f0ec953c86735bbe2081ac738865 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region edf3600642cf1af12a36dc726f301ef2 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 24857182eb0da41497c7febd6e0c060d changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9bab6e4180453738e691368cc5fbb429 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b2697dde157dfed90cc8595ff82dc081 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d2b53404c106b2f9960c2de982a64c06 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bc9bead55355bc7d07a869ba01fa7404 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 668c3a8f00f6512df533713b2c2d46cb changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 409ba30f82abbec05bb6c84a023d1e9a changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 026ea48c246e26b4197a66fa854f5022 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1f231052a3b0401ad15e94c5bf55a972 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8ebd6e15e1eb3717f4dd8f1695d2b691 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 4db6e838ae3d565bc2af54adcdf3a3c0 changed from -1.0 to 0.0, refreshing cache 2024-11-13T14:13:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-13T14:13:44,326 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-13T14:13:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-13T14:13:44,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BalancerClusterState(204): Hosts are {eb1966fd6578=0} racks are {/default-rack=0} 2024-11-13T14:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-13T14:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-13T14:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-13T14:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T14:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T14:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T14:13:44,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T14:13:44,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T14:13:44,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T14:13:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-13T14:13:44,334 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-13T14:13:44,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-13T14:13:44,337 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T14:13:44,347 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-13T14:13:44,372 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=381, OpenFileDescriptor=647, MaxFileDescriptor=1048576, SystemLoadAverage=291, ProcessCount=11, AvailableMemoryMB=3213 2024-11-13T14:13:44,376 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-13T14:13:44,377 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-13T14:13:44,377 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing eb1966fd6578,38145,1731507219374 2024-11-13T14:13:44,378 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server eb1966fd6578,38145,1731507219374: Simulated kill ***** 2024-11-13T14:13:44,378 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T14:13:44,380 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T14:13:44,383 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T14:13:44,383 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T14:13:44,384 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T14:13:44,384 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 298472544 }, "NonHeapMemoryUsage": { "committed": 151846912, "init": 7667712, "max": -1, "used": 149470120 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "eb1966fd6578" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "eb1966fd6578" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "eb1966fd6578" } ] } 2024-11-13T14:13:44,388 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36813 {}] master.MasterRpcServices(700): eb1966fd6578,38145,1731507219374 reported a fatal error: ***** ABORTING region server eb1966fd6578,38145,1731507219374: Simulated kill ***** 2024-11-13T14:13:44,390 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eb1966fd6578,38145,1731507219374' ***** 2024-11-13T14:13:44,391 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-13T14:13:44,391 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T14:13:44,391 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T14:13:44,391 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T14:13:44,392 INFO [RS:0;eb1966fd6578:38145 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T14:13:44,392 INFO [RS:0;eb1966fd6578:38145 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T14:13:44,392 INFO [RS:0;eb1966fd6578:38145 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T14:13:44,392 INFO [RS:0;eb1966fd6578:38145 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;eb1966fd6578:38145. 2024-11-13T14:13:44,392 DEBUG [RS:0;eb1966fd6578:38145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T14:13:44,397 DEBUG [RS:0;eb1966fd6578:38145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T14:13:44,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741836_1012 (size=32114) 2024-11-13T14:13:44,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741836_1012 (size=32114) 2024-11-13T14:13:44,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741836_1012 (size=32114) 2024-11-13T14:13:44,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741834_1010 (size=2237) 2024-11-13T14:13:44,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741834_1010 (size=2237) 2024-11-13T14:13:44,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741834_1010 (size=2237) 2024-11-13T14:13:44,408 DEBUG [RS:0;eb1966fd6578:38145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T14:13:44,409 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T14:13:44,409 INFO [RS:0;eb1966fd6578:38145 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38145 2024-11-13T14:13:44,435 INFO [regionserver/eb1966fd6578:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T14:13:44,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/eb1966fd6578,38145,1731507219374 2024-11-13T14:13:44,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T14:13:44,486 INFO [RS:0;eb1966fd6578:38145 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T14:13:44,497 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eb1966fd6578,38145,1731507219374] 2024-11-13T14:13:44,507 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/eb1966fd6578,38145,1731507219374 already deleted, retry=false 2024-11-13T14:13:44,507 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of eb1966fd6578,38145,1731507219374 on eb1966fd6578,36813,1731507218489 2024-11-13T14:13:44,513 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:44,517 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=57 for eb1966fd6578,38145,1731507219374 (carryingMeta=true) eb1966fd6578,38145,1731507219374/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@1bec0139[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T14:13:44,518 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:44,520 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true, isMeta: true 2024-11-13T14:13:44,522 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting 2024-11-13T14:13:44,523 INFO [PEWorker-1 {}] master.SplitWALManager(105): eb1966fd6578,38145,1731507219374 WAL count=1, meta=true 2024-11-13T14:13:44,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta}] 2024-11-13T14:13:44,530 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:44,532 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:44,593 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T14:13:44,596 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T14:13:44,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38145-0x101348b60910001, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T14:13:44,597 INFO [RS:0;eb1966fd6578:38145 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T14:13:44,597 INFO [RS:0;eb1966fd6578:38145 {}] regionserver.HRegionServer(1031): Exiting; stopping=eb1966fd6578,38145,1731507219374; zookeeper connection closed. 2024-11-13T14:13:44,597 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@36987346 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@36987346 2024-11-13T14:13:44,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42135 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-13T14:13:44,729 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, size=31.4 K (32114bytes) 2024-11-13T14:13:44,730 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T14:13:44,730 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T14:13:44,730 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta 2024-11-13T14:13:44,736 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta after 3ms 2024-11-13T14:13:44,744 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta took 17ms 2024-11-13T14:13:44,754 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-13T14:13:44,759 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta.temp 2024-11-13T14:13:44,762 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta.temp 2024-11-13T14:13:44,766 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta so closing down 2024-11-13T14:13:44,766 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T14:13:44,766 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T14:13:44,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741864_1040 (size=31873) 2024-11-13T14:13:44,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741864_1040 (size=31873) 2024-11-13T14:13:44,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741864_1040 (size=31873) 2024-11-13T14:13:44,783 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-13T14:13:44,786 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta.temp to hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T14:13:44,790 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-13T14:13:44,791 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, journal: Splitting hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, size=31.4 K (32114bytes) at 1731507224729Creating recovered edits writer path=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta.temp at 1731507224762 (+33 ms)Finishing writing output for hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta so closing down at 1731507224766 (+4 ms)3 split writer threads finished at 1731507224766Closed recovered edits writer path=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1731507224783 (+17 ms)Rename recovered edits hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta.temp to hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731507224786 (+3 ms)Processed 58 edits across 1 Regions in 41 ms; skipped=1; WAL=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1731507224791 (+5 ms) 2024-11-13T14:13:44,791 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta 2024-11-13T14:13:44,793 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-13T14:13:44,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-13T14:13:44,802 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta to hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/oldWALs 2024-11-13T14:13:44,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-13T14:13:44,805 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 in 270 msec 2024-11-13T14:13:44,806 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:44,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-13T14:13:44,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 in 282 msec 2024-11-13T14:13:44,813 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting dir is empty, no logs to split. 2024-11-13T14:13:44,813 INFO [PEWorker-1 {}] master.SplitWALManager(105): eb1966fd6578,38145,1731507219374 WAL count=0, meta=true 2024-11-13T14:13:44,813 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if eb1966fd6578,38145,1731507219374 WAL splitting is done? wals=0, meta=true 2024-11-13T14:13:44,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T14:13:44,816 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:44,817 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T14:13:44,968 DEBUG [eb1966fd6578:36813 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T14:13:44,968 INFO [regionserver/eb1966fd6578:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T14:13:44,968 INFO [regionserver/eb1966fd6578:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T14:13:44,968 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(204): Hosts are {eb1966fd6578=0} racks are {/default-rack=0} 2024-11-13T14:13:44,968 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T14:13:44,968 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T14:13:44,968 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T14:13:44,968 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T14:13:44,968 INFO [eb1966fd6578:36813 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T14:13:44,968 INFO [eb1966fd6578:36813 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T14:13:44,968 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T14:13:44,969 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:44,970 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPENING 2024-11-13T14:13:45,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:45,181 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:45,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:45,181 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:45,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:45,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:45,181 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:45,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:45,340 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T14:13:45,340 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T14:13:45,341 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T14:13:45,344 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=eb1966fd6578%2C42135%2C1731507219475.meta, suffix=.meta, logDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,42135,1731507219475, archiveDir=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/oldWALs, maxLogs=32 2024-11-13T14:13:45,364 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,42135,1731507219475/eb1966fd6578%2C42135%2C1731507219475.meta.1731507225346.meta, exclude list is [], retry=0 2024-11-13T14:13:45,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:35125,DS-7d222e8a-09d0-41ad-8291-a32be5623236,DISK] 2024-11-13T14:13:45,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46747,DS-0b2f9f3b-9824-4059-870f-9bb615a80b01,DISK] 2024-11-13T14:13:45,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33015,DS-9b4f3d38-26ab-4b10-b6b8-307e24d8051e,DISK] 2024-11-13T14:13:45,370 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,42135,1731507219475/eb1966fd6578%2C42135%2C1731507219475.meta.1731507225346.meta 2024-11-13T14:13:45,371 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43485:43485),(127.0.0.1/127.0.0.1:42835:42835),(127.0.0.1/127.0.0.1:39941:39941)] 2024-11-13T14:13:45,371 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T14:13:45,372 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T14:13:45,372 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T14:13:45,372 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T14:13:45,372 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T14:13:45,372 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:45,372 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T14:13:45,372 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T14:13:45,374 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T14:13:45,375 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T14:13:45,375 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:45,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:45,376 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T14:13:45,377 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T14:13:45,377 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:45,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:45,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T14:13:45,380 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T14:13:45,380 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:45,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:45,380 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T14:13:45,381 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T14:13:45,381 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:45,382 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T14:13:45,382 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T14:13:45,383 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740 2024-11-13T14:13:45,385 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740 2024-11-13T14:13:45,386 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T14:13:45,399 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T14:13:45,402 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-13T14:13:45,473 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/info/96378a728d224c79b5f3dfbac77260e3 is 138, key is test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06./info:regioninfo/1731507223115/Put/seqid=0 2024-11-13T14:13:45,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741866_1042 (size=34924) 2024-11-13T14:13:45,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741866_1042 (size=34924) 2024-11-13T14:13:45,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741866_1042 (size=34924) 2024-11-13T14:13:45,486 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/info/96378a728d224c79b5f3dfbac77260e3 2024-11-13T14:13:45,549 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/ns/4a5bc71ba97042c6b8b1ef9ed201431e is 43, key is default/ns:d/1731507221704/Put/seqid=0 2024-11-13T14:13:45,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741867_1043 (size=5153) 2024-11-13T14:13:45,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741867_1043 (size=5153) 2024-11-13T14:13:45,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741867_1043 (size=5153) 2024-11-13T14:13:45,556 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/ns/4a5bc71ba97042c6b8b1ef9ed201431e 2024-11-13T14:13:45,582 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/table/08adca1cc2df4865957ddc416edc8aa2 is 40, key is test/table:state/1731507223561/Put/seqid=0 2024-11-13T14:13:45,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741868_1044 (size=5165) 2024-11-13T14:13:45,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741868_1044 (size=5165) 2024-11-13T14:13:45,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741868_1044 (size=5165) 2024-11-13T14:13:45,591 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/table/08adca1cc2df4865957ddc416edc8aa2 2024-11-13T14:13:45,601 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/info/96378a728d224c79b5f3dfbac77260e3 as hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/info/96378a728d224c79b5f3dfbac77260e3 2024-11-13T14:13:45,611 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/info/96378a728d224c79b5f3dfbac77260e3, entries=260, sequenceid=60, filesize=34.1 K 2024-11-13T14:13:45,615 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/ns/4a5bc71ba97042c6b8b1ef9ed201431e as hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/ns/4a5bc71ba97042c6b8b1ef9ed201431e 2024-11-13T14:13:45,624 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/ns/4a5bc71ba97042c6b8b1ef9ed201431e, entries=2, sequenceid=60, filesize=5.0 K 2024-11-13T14:13:45,625 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/table/08adca1cc2df4865957ddc416edc8aa2 as hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/table/08adca1cc2df4865957ddc416edc8aa2 2024-11-13T14:13:45,634 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/table/08adca1cc2df4865957ddc416edc8aa2, entries=2, sequenceid=60, filesize=5.0 K 2024-11-13T14:13:45,639 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 235ms, sequenceid=60, compaction requested=false; wal=null 2024-11-13T14:13:45,641 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T14:13:45,644 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-13T14:13:45,646 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T14:13:45,646 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T14:13:45,647 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T14:13:45,649 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T14:13:45,652 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-13T14:13:45,654 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73061389, jitterRate=0.08869953453540802}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T14:13:45,654 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T14:13:45,654 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731507225372Writing region info on filesystem at 1731507225372Initializing all the Stores at 1731507225374 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731507225374Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731507225374Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507225374Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731507225374Obtaining lock to block concurrent updates at 1731507225402 (+28 ms)Preparing flush snapshotting stores in 1588230740 at 1731507225403 (+1 ms)Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1731507225405 (+2 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731507225405Flushing 1588230740/info: creating writer at 1731507225406 (+1 ms)Flushing 1588230740/info: appending metadata at 1731507225462 (+56 ms)Flushing 1588230740/info: closing flushed file at 1731507225465 (+3 ms)Flushing 1588230740/ns: creating writer at 1731507225532 (+67 ms)Flushing 1588230740/ns: appending metadata at 1731507225547 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731507225547Flushing 1588230740/table: creating writer at 1731507225565 (+18 ms)Flushing 1588230740/table: appending metadata at 1731507225581 (+16 ms)Flushing 1588230740/table: closing flushed file at 1731507225581Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@680dde7f: reopening flushed file at 1731507225600 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@25dbbb0a: reopening flushed file at 1731507225613 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@710d2535: reopening flushed file at 1731507225624 (+11 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 235ms, sequenceid=60, compaction requested=false; wal=null at 1731507225639 (+15 ms)Cleaning up temporary data from old regions at 1731507225646 (+7 ms)Running coprocessor post-open hooks at 1731507225654 (+8 ms)Region opened successfully at 1731507225654 2024-11-13T14:13:45,655 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731507225334 2024-11-13T14:13:45,658 DEBUG [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T14:13:45,658 INFO [RS_OPEN_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T14:13:45,659 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:45,660 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:45,673 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-13T14:13:45,707 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:45,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:45,707 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:45,707 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:45,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:45,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:45,707 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:45,709 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-13T14:13:45,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 in 526 msec 2024-11-13T14:13:45,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-13T14:13:45,712 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 895 msec 2024-11-13T14:13:45,712 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): eb1966fd6578,38145,1731507219374 had 9 regions 2024-11-13T14:13:45,713 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true, isMeta: false 2024-11-13T14:13:45,716 INFO [PEWorker-1 {}] master.SplitWALManager(105): eb1966fd6578,38145,1731507219374 WAL count=1, meta=false 2024-11-13T14:13:45,716 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090}] 2024-11-13T14:13:45,718 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:45,720 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:45,874 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41771 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-13T14:13:45,894 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090, size=2.2 K (2237bytes) 2024-11-13T14:13:45,894 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090 2024-11-13T14:13:45,895 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090 after 0ms 2024-11-13T14:13:45,898 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090 took 5ms 2024-11-13T14:13:45,902 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ee5b69e1e01798b1055c61a7885f87b4: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-13T14:13:45,904 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 1d3783ebc90fc59392fa002c53f7c56f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-13T14:13:45,906 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 2027122fd77a73d52ce4d3368dd23e0a: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-13T14:13:45,908 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ef7ba06bec150589d1c840bbd1b13d9b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-13T14:13:45,910 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for def71812430057c99b7be5281021b22c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-13T14:13:45,912 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for edbfee0cdbcdc3896eb3a9b618149bc3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-13T14:13:45,914 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for eb1bb70b6e4e99eeece01c54dd8a665c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-13T14:13:45,916 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for dd3915f2bd1ed9356a65d71ba154c88b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-13T14:13:45,916 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090 so closing down 2024-11-13T14:13:45,916 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T14:13:45,916 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T14:13:45,916 INFO [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 17 ms; skipped=8; WAL=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-13T14:13:45,916 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090, journal: Splitting hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090, size=2.2 K (2237bytes) at 1731507225894Finishing writing output for hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090 so closing down at 1731507225916 (+22 ms)3 split writer threads finished at 1731507225916Processed 8 edits across 0 Regions in 17 ms; skipped=8; WAL=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1731507225916 2024-11-13T14:13:45,917 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090 2024-11-13T14:13:45,917 DEBUG [RS_LOG_REPLAY_OPS-regionserver/eb1966fd6578:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-13T14:13:45,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36813 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-13T14:13:45,921 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting/eb1966fd6578%2C38145%2C1731507219374.1731507221090 to hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/oldWALs 2024-11-13T14:13:45,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-13T14:13:45,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 in 203 msec 2024-11-13T14:13:45,925 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:45,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-13T14:13:45,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 in 210 msec 2024-11-13T14:13:45,930 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/WALs/eb1966fd6578,38145,1731507219374-splitting dir is empty, no logs to split. 2024-11-13T14:13:45,931 INFO [PEWorker-1 {}] master.SplitWALManager(105): eb1966fd6578,38145,1731507219374 WAL count=0, meta=false 2024-11-13T14:13:45,931 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if eb1966fd6578,38145,1731507219374 WAL splitting is done? wals=0, meta=false 2024-11-13T14:13:45,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN}] 2024-11-13T14:13:45,935 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:45,936 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T14:13:45,937 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:45,938 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T14:13:45,939 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:45,940 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T14:13:45,941 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:45,942 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T14:13:45,943 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:45,944 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T14:13:45,945 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:45,945 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T14:13:45,946 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:45,947 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T14:13:45,949 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:45,950 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T14:13:46,086 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(204): Hosts are {eb1966fd6578=0} racks are {/default-rack=0} 2024-11-13T14:13:46,087 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T14:13:46,087 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T14:13:46,087 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T14:13:46,087 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T14:13:46,087 INFO [eb1966fd6578:36813 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T14:13:46,087 INFO [eb1966fd6578:36813 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T14:13:46,087 DEBUG [eb1966fd6578:36813 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T14:13:46,088 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,089 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,089 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,090 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=def71812430057c99b7be5281021b22c, regionState=OPENING, regionLocation=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,090 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server eb1966fd6578:38145 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:13:46,090 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,091 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,091 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,092 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1, error=java.net.ConnectException: Call to address=eb1966fd6578:38145 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 2024-11-13T14:13:46,092 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, regionLocation=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T14:13:46,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1 from cache 2024-11-13T14:13:46,094 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address eb1966fd6578:38145 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 2024-11-13T14:13:46,095 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server eb1966fd6578:38145 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T14:13:46,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=eb1966fd6578:38145 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 2024-11-13T14:13:46,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=eb1966fd6578:38145 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 2024-11-13T14:13:46,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=eb1966fd6578:38145 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 2024-11-13T14:13:46,096 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=eb1966fd6578:38145 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 2024-11-13T14:13:46,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=eb1966fd6578:38145 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 2024-11-13T14:13:46,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=eb1966fd6578:38145 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 2024-11-13T14:13:46,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,38145,1731507219374, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=eb1966fd6578:38145 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 2024-11-13T14:13:46,097 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address eb1966fd6578:38145 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: eb1966fd6578/172.17.0.2:38145 2024-11-13T14:13:46,195 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T14:13:46,195 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=eb1966fd6578,42135,1731507219475, seqNum=-1] 2024-11-13T14:13:46,196 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T14:13:46,197 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40277, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T14:13:46,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN because future has completed 2024-11-13T14:13:46,200 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:46,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN because future has completed 2024-11-13T14:13:46,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:46,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN because future has completed 2024-11-13T14:13:46,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN because future has completed 2024-11-13T14:13:46,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN because future has completed 2024-11-13T14:13:46,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN because future has completed 2024-11-13T14:13:46,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:46,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN because future has completed 2024-11-13T14:13:46,214 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN because future has completed 2024-11-13T14:13:46,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:46,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:46,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,41771,1731507219528}] 2024-11-13T14:13:46,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure def71812430057c99b7be5281021b22c, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:46,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbfee0cdbcdc3896eb3a9b618149bc3, server=eb1966fd6578,42135,1731507219475}] 2024-11-13T14:13:46,356 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:46,357 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => dd3915f2bd1ed9356a65d71ba154c88b, NAME => 'test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-13T14:13:46,357 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:46,357 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:46,357 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:46,357 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:46,359 INFO [StoreOpener-dd3915f2bd1ed9356a65d71ba154c88b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:46,361 INFO [StoreOpener-dd3915f2bd1ed9356a65d71ba154c88b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dd3915f2bd1ed9356a65d71ba154c88b columnFamilyName family 2024-11-13T14:13:46,361 DEBUG [StoreOpener-dd3915f2bd1ed9356a65d71ba154c88b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:46,361 INFO [StoreOpener-dd3915f2bd1ed9356a65d71ba154c88b-1 {}] regionserver.HStore(327): Store=dd3915f2bd1ed9356a65d71ba154c88b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:46,362 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:46,363 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:46,364 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:46,365 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:46,365 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:46,368 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:46,369 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened dd3915f2bd1ed9356a65d71ba154c88b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69770842, jitterRate=0.039666563272476196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:46,369 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:46,369 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for dd3915f2bd1ed9356a65d71ba154c88b: Running coprocessor pre-open hook at 1731507226357Writing region info on filesystem at 1731507226358 (+1 ms)Initializing all the Stores at 1731507226359 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507226359Cleaning up temporary data from old regions at 1731507226365 (+6 ms)Running coprocessor post-open hooks at 1731507226369 (+4 ms)Region opened successfully at 1731507226369 2024-11-13T14:13:46,370 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:46,370 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => eb1bb70b6e4e99eeece01c54dd8a665c, NAME => 'test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-13T14:13:46,371 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:46,371 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:46,371 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:46,371 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:46,372 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b., pid=73, masterSystemTime=1731507226352 2024-11-13T14:13:46,373 INFO [StoreOpener-eb1bb70b6e4e99eeece01c54dd8a665c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:46,375 INFO [StoreOpener-eb1bb70b6e4e99eeece01c54dd8a665c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region eb1bb70b6e4e99eeece01c54dd8a665c columnFamilyName family 2024-11-13T14:13:46,375 DEBUG [StoreOpener-eb1bb70b6e4e99eeece01c54dd8a665c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:46,376 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:46,376 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:46,376 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:46,376 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => def71812430057c99b7be5281021b22c, NAME => 'test,ggg,1731507221993.def71812430057c99b7be5281021b22c.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-13T14:13:46,376 INFO [StoreOpener-eb1bb70b6e4e99eeece01c54dd8a665c-1 {}] regionserver.HStore(327): Store=eb1bb70b6e4e99eeece01c54dd8a665c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:46,376 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:46,377 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test def71812430057c99b7be5281021b22c 2024-11-13T14:13:46,377 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,ggg,1731507221993.def71812430057c99b7be5281021b22c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:46,377 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for def71812430057c99b7be5281021b22c 2024-11-13T14:13:46,377 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for def71812430057c99b7be5281021b22c 2024-11-13T14:13:46,377 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:46,378 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-13T14:13:46,378 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,379 INFO [StoreOpener-def71812430057c99b7be5281021b22c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region def71812430057c99b7be5281021b22c 2024-11-13T14:13:46,379 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:46,380 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:46,380 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:46,380 INFO [StoreOpener-def71812430057c99b7be5281021b22c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region def71812430057c99b7be5281021b22c columnFamilyName family 2024-11-13T14:13:46,380 DEBUG [StoreOpener-def71812430057c99b7be5281021b22c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:46,379 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,381 INFO [StoreOpener-def71812430057c99b7be5281021b22c-1 {}] regionserver.HStore(327): Store=def71812430057c99b7be5281021b22c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:46,382 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for def71812430057c99b7be5281021b22c 2024-11-13T14:13:46,383 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:46,383 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/def71812430057c99b7be5281021b22c 2024-11-13T14:13:46,384 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened eb1bb70b6e4e99eeece01c54dd8a665c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60898685, jitterRate=-0.09253887832164764}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:46,384 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:46,384 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for eb1bb70b6e4e99eeece01c54dd8a665c: Running coprocessor pre-open hook at 1731507226371Writing region info on filesystem at 1731507226371Initializing all the Stores at 1731507226373 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507226373Cleaning up temporary data from old regions at 1731507226380 (+7 ms)Running coprocessor post-open hooks at 1731507226384 (+4 ms)Region opened successfully at 1731507226384 2024-11-13T14:13:46,385 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/def71812430057c99b7be5281021b22c 2024-11-13T14:13:46,386 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c., pid=75, masterSystemTime=1731507226365 2024-11-13T14:13:46,387 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for def71812430057c99b7be5281021b22c 2024-11-13T14:13:46,387 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for def71812430057c99b7be5281021b22c 2024-11-13T14:13:46,389 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:46,389 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:46,389 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:46,389 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 1d3783ebc90fc59392fa002c53f7c56f, NAME => 'test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-13T14:13:46,390 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:46,390 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:46,390 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:46,390 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:46,391 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for def71812430057c99b7be5281021b22c 2024-11-13T14:13:46,393 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-13T14:13:46,396 INFO [StoreOpener-1d3783ebc90fc59392fa002c53f7c56f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:46,397 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,397 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbfee0cdbcdc3896eb3a9b618149bc3, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,397 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened def71812430057c99b7be5281021b22c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60644423, jitterRate=-0.09632767736911774}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:46,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,397 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for def71812430057c99b7be5281021b22c 2024-11-13T14:13:46,397 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for def71812430057c99b7be5281021b22c: Running coprocessor pre-open hook at 1731507226377Writing region info on filesystem at 1731507226377Initializing all the Stores at 1731507226378 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507226378Cleaning up temporary data from old regions at 1731507226388 (+10 ms)Running coprocessor post-open hooks at 1731507226397 (+9 ms)Region opened successfully at 1731507226397 2024-11-13T14:13:46,398 INFO [StoreOpener-1d3783ebc90fc59392fa002c53f7c56f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1d3783ebc90fc59392fa002c53f7c56f columnFamilyName family 2024-11-13T14:13:46,398 DEBUG [StoreOpener-1d3783ebc90fc59392fa002c53f7c56f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:46,398 INFO [StoreOpener-1d3783ebc90fc59392fa002c53f7c56f-1 {}] regionserver.HStore(327): Store=1d3783ebc90fc59392fa002c53f7c56f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:46,399 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731507221993.def71812430057c99b7be5281021b22c., pid=78, masterSystemTime=1731507226352 2024-11-13T14:13:46,399 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:46,399 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbfee0cdbcdc3896eb3a9b618149bc3, server=eb1966fd6578,42135,1731507219475 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,400 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:46,402 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:46,403 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:46,403 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:46,403 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:46,403 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:46,403 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:46,403 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => ee5b69e1e01798b1055c61a7885f87b4, NAME => 'test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-13T14:13:46,404 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:46,404 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:46,404 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:46,404 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:46,406 INFO [StoreOpener-ee5b69e1e01798b1055c61a7885f87b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:46,407 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:46,408 INFO [StoreOpener-ee5b69e1e01798b1055c61a7885f87b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ee5b69e1e01798b1055c61a7885f87b4 columnFamilyName family 2024-11-13T14:13:46,408 DEBUG [StoreOpener-ee5b69e1e01798b1055c61a7885f87b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:46,409 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 1d3783ebc90fc59392fa002c53f7c56f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63434075, jitterRate=-0.054758623242378235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:46,409 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:46,409 INFO [StoreOpener-ee5b69e1e01798b1055c61a7885f87b4-1 {}] regionserver.HStore(327): Store=ee5b69e1e01798b1055c61a7885f87b4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:46,409 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 1d3783ebc90fc59392fa002c53f7c56f: Running coprocessor pre-open hook at 1731507226390Writing region info on filesystem at 1731507226390Initializing all the Stores at 1731507226391 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507226391Cleaning up temporary data from old regions at 1731507226403 (+12 ms)Running coprocessor post-open hooks at 1731507226409 (+6 ms)Region opened successfully at 1731507226409 2024-11-13T14:13:46,409 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:46,410 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:46,410 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f., pid=74, masterSystemTime=1731507226365 2024-11-13T14:13:46,413 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:46,414 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:46,414 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:46,414 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:46,414 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:46,414 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:46,414 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => ef7ba06bec150589d1c840bbd1b13d9b, NAME => 'test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-13T14:13:46,415 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:46,415 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:46,415 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:46,415 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:46,416 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:46,417 INFO [StoreOpener-ef7ba06bec150589d1c840bbd1b13d9b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:46,417 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened ee5b69e1e01798b1055c61a7885f87b4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61307379, jitterRate=-0.08644886314868927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:46,418 INFO [StoreOpener-ef7ba06bec150589d1c840bbd1b13d9b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ef7ba06bec150589d1c840bbd1b13d9b columnFamilyName family 2024-11-13T14:13:46,418 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:46,418 DEBUG [StoreOpener-ef7ba06bec150589d1c840bbd1b13d9b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:46,418 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for ee5b69e1e01798b1055c61a7885f87b4: Running coprocessor pre-open hook at 1731507226404Writing region info on filesystem at 1731507226404Initializing all the Stores at 1731507226405 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507226405Cleaning up temporary data from old regions at 1731507226414 (+9 ms)Running coprocessor post-open hooks at 1731507226418 (+4 ms)Region opened successfully at 1731507226418 2024-11-13T14:13:46,418 INFO [StoreOpener-ef7ba06bec150589d1c840bbd1b13d9b-1 {}] regionserver.HStore(327): Store=ef7ba06bec150589d1c840bbd1b13d9b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:46,418 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:46,419 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4., pid=72, masterSystemTime=1731507226352 2024-11-13T14:13:46,419 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:46,421 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:46,421 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:46,421 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:46,422 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:46,422 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:46,422 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:46,422 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => edbfee0cdbcdc3896eb3a9b618149bc3, NAME => 'test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-13T14:13:46,422 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:46,422 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:46,422 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:46,422 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:46,424 INFO [StoreOpener-edbfee0cdbcdc3896eb3a9b618149bc3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:46,424 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:46,425 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened ef7ba06bec150589d1c840bbd1b13d9b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64289539, jitterRate=-0.04201121628284454}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:46,425 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:46,425 INFO [StoreOpener-edbfee0cdbcdc3896eb3a9b618149bc3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region edbfee0cdbcdc3896eb3a9b618149bc3 columnFamilyName family 2024-11-13T14:13:46,425 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for ef7ba06bec150589d1c840bbd1b13d9b: Running coprocessor pre-open hook at 1731507226415Writing region info on filesystem at 1731507226415Initializing all the Stores at 1731507226416 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507226416Cleaning up temporary data from old regions at 1731507226422 (+6 ms)Running coprocessor post-open hooks at 1731507226425 (+3 ms)Region opened successfully at 1731507226425 2024-11-13T14:13:46,425 DEBUG [StoreOpener-edbfee0cdbcdc3896eb3a9b618149bc3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:46,426 INFO [StoreOpener-edbfee0cdbcdc3896eb3a9b618149bc3-1 {}] regionserver.HStore(327): Store=edbfee0cdbcdc3896eb3a9b618149bc3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:46,426 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b., pid=76, masterSystemTime=1731507226365 2024-11-13T14:13:46,426 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:46,427 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:46,429 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:46,429 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:46,429 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:46,429 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:46,429 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 2027122fd77a73d52ce4d3368dd23e0a, NAME => 'test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-13T14:13:46,429 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:46,429 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:46,430 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:46,430 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T14:13:46,430 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:46,430 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:46,431 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:46,431 INFO [StoreOpener-2027122fd77a73d52ce4d3368dd23e0a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:46,432 INFO [StoreOpener-2027122fd77a73d52ce4d3368dd23e0a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2027122fd77a73d52ce4d3368dd23e0a columnFamilyName family 2024-11-13T14:13:46,432 DEBUG [StoreOpener-2027122fd77a73d52ce4d3368dd23e0a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T14:13:46,432 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened edbfee0cdbcdc3896eb3a9b618149bc3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74107455, jitterRate=0.10428713262081146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:46,433 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:46,433 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for edbfee0cdbcdc3896eb3a9b618149bc3: Running coprocessor pre-open hook at 1731507226422Writing region info on filesystem at 1731507226422Initializing all the Stores at 1731507226423 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507226423Cleaning up temporary data from old regions at 1731507226429 (+6 ms)Running coprocessor post-open hooks at 1731507226433 (+4 ms)Region opened successfully at 1731507226433 2024-11-13T14:13:46,433 INFO [StoreOpener-2027122fd77a73d52ce4d3368dd23e0a-1 {}] regionserver.HStore(327): Store=2027122fd77a73d52ce4d3368dd23e0a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T14:13:46,433 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:46,434 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3., pid=79, masterSystemTime=1731507226352 2024-11-13T14:13:46,434 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:46,436 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:46,436 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:46,436 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:46,437 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:46,437 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:46,440 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:46,441 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 2027122fd77a73d52ce4d3368dd23e0a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71507878, jitterRate=0.06555041670799255}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T14:13:46,441 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:46,441 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 2027122fd77a73d52ce4d3368dd23e0a: Running coprocessor pre-open hook at 1731507226430Writing region info on filesystem at 1731507226430Initializing all the Stores at 1731507226431 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731507226431Cleaning up temporary data from old regions at 1731507226437 (+6 ms)Running coprocessor post-open hooks at 1731507226441 (+4 ms)Region opened successfully at 1731507226441 2024-11-13T14:13:46,442 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a., pid=77, masterSystemTime=1731507226365 2024-11-13T14:13:46,445 DEBUG [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:46,445 INFO [RS_OPEN_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:46,497 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,497 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,497 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,498 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,498 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,498 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,498 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,498 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,499 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,499 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,507 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,508 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,509 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,510 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,512 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,518 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,519 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure def71812430057c99b7be5281021b22c, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbfee0cdbcdc3896eb3a9b618149bc3, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,520 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,520 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,521 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,521 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,521 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,522 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,522 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,522 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,522 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure def71812430057c99b7be5281021b22c, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,522 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure edbfee0cdbcdc3896eb3a9b618149bc3, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,522 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 22 msec 2024-11-13T14:13:46,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,525 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,525 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,574 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,574 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,574 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,575 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN to state=OFFLINE, location=null, table=test, region=def71812430057c99b7be5281021b22c to restore RIT 2024-11-13T14:13:46,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN to state=OFFLINE, location=null, table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3 to restore RIT 2024-11-13T14:13:46,577 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN to state=OFFLINE, location=null, table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c to restore RIT 2024-11-13T14:13:46,580 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256580 2024-11-13T14:13:46,581 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,580 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,583 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,584 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,584 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure def71812430057c99b7be5281021b22c, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,584 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,584 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure def71812430057c99b7be5281021b22c, server=eb1966fd6578,42135,1731507219475 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,609 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,610 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,611 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,612 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,613 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,614 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:46,614 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507286614 2024-11-13T14:13:46,614 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507346614 2024-11-13T14:13:46,614 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 33 msec 2024-11-13T14:13:46,614 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,614 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,614 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,614 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,614 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,615 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,615 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,615 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,616 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,620 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,621 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,621 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,622 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,625 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure def71812430057c99b7be5281021b22c, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,626 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,626 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,627 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,627 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,627 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,627 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,627 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,627 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,627 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,628 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,628 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,628 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure def71812430057c99b7be5281021b22c, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,628 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 12 msec 2024-11-13T14:13:46,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,629 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,629 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,637 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,637 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,637 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN to state=OFFLINE, location=null, table=test, region=def71812430057c99b7be5281021b22c to restore RIT 2024-11-13T14:13:46,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN to state=OFFLINE, location=null, table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3 to restore RIT 2024-11-13T14:13:46,638 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN to state=OFFLINE, location=null, table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c to restore RIT 2024-11-13T14:13:46,639 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256639 2024-11-13T14:13:46,639 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,639 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,640 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,646 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,646 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,646 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,646 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,41771,1731507219528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,653 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,654 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,654 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:46,654 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507286654 2024-11-13T14:13:46,654 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507346654 2024-11-13T14:13:46,654 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 15 msec 2024-11-13T14:13:46,654 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,654 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,654 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,654 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,654 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,654 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,654 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,655 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,655 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,659 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,660 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,660 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,660 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,660 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,660 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,661 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,663 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,663 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,663 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,663 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,664 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,665 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,665 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,665 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,665 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,665 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,665 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,665 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,665 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,665 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,666 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2027122fd77a73d52ce4d3368dd23e0a, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,666 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-13T14:13:46,667 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,667 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,667 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,679 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,679 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,679 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,680 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,680 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,680 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,680 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,680 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,680 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN to state=OFFLINE, location=null, table=test, region=def71812430057c99b7be5281021b22c to restore RIT 2024-11-13T14:13:46,680 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN to state=OFFLINE, location=null, table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3 to restore RIT 2024-11-13T14:13:46,681 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN to state=OFFLINE, location=null, table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c to restore RIT 2024-11-13T14:13:46,681 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256681 2024-11-13T14:13:46,681 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,681 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,682 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,685 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,685 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,685 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,685 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,41771,1731507219528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,694 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,695 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,696 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:46,696 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507286696 2024-11-13T14:13:46,696 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507346696 2024-11-13T14:13:46,696 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-13T14:13:46,696 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,696 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,696 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,696 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,696 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,696 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,697 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,697 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,698 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,701 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,702 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,702 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,702 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,702 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,702 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,702 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,702 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,702 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,702 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,703 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,705 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,706 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,707 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,708 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,708 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,708 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,708 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,708 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,708 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,708 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,708 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,708 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure ef7ba06bec150589d1c840bbd1b13d9b, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,709 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-13T14:13:46,710 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,710 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,710 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,721 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,721 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN to state=OFFLINE, location=null, table=test, region=def71812430057c99b7be5281021b22c to restore RIT 2024-11-13T14:13:46,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN to state=OFFLINE, location=null, table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3 to restore RIT 2024-11-13T14:13:46,723 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN to state=OFFLINE, location=null, table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c to restore RIT 2024-11-13T14:13:46,723 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256723 2024-11-13T14:13:46,724 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,724 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,725 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,728 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,728 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,729 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,729 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,732 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,733 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,734 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,735 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,736 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:46,736 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507286736 2024-11-13T14:13:46,736 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507346736 2024-11-13T14:13:46,736 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-13T14:13:46,736 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,736 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,736 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,736 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,737 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,737 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,737 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,737 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,738 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,741 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,742 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,746 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,746 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,746 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,746 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,746 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,747 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,747 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,747 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,747 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,747 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,747 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,747 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,747 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,747 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,747 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,747 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure eb1bb70b6e4e99eeece01c54dd8a665c, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,747 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-13T14:13:46,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,748 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,748 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,759 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,759 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,760 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,760 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,760 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,761 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,761 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,761 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,761 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,761 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,761 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN to state=OFFLINE, location=null, table=test, region=def71812430057c99b7be5281021b22c to restore RIT 2024-11-13T14:13:46,761 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN to state=OFFLINE, location=null, table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3 to restore RIT 2024-11-13T14:13:46,761 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN to state=OFFLINE, location=null, table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c to restore RIT 2024-11-13T14:13:46,762 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256762 2024-11-13T14:13:46,762 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,762 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,763 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,767 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,767 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,768 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,770 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,771 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,772 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,773 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,774 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:46,774 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507286774 2024-11-13T14:13:46,774 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507346774 2024-11-13T14:13:46,774 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-13T14:13:46,774 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,774 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,774 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,774 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,774 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,775 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,775 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,779 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,780 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,782 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,784 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,785 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,786 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,786 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,787 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1d3783ebc90fc59392fa002c53f7c56f, server=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,787 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-13T14:13:46,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,788 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,788 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,795 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,795 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,796 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,796 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,796 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,796 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,796 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,796 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN to state=OFFLINE, location=null, table=test, region=def71812430057c99b7be5281021b22c to restore RIT 2024-11-13T14:13:46,796 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN to state=OFFLINE, location=null, table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3 to restore RIT 2024-11-13T14:13:46,796 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN to state=OFFLINE, location=null, table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c to restore RIT 2024-11-13T14:13:46,797 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256797 2024-11-13T14:13:46,797 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,797 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,798 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,800 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,800 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,800 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,800 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,808 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,809 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,810 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:46,810 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507286810 2024-11-13T14:13:46,810 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507346810 2024-11-13T14:13:46,810 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-13T14:13:46,810 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,810 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,810 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,810 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,811 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,811 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,811 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,811 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,812 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,815 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,816 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,816 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,818 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,819 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,819 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,819 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,820 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,820 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,820 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,820 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-13T14:13:46,821 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,821 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,821 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,833 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,833 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,833 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,833 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,834 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,834 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,834 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,834 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,834 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,834 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,834 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN to state=OFFLINE, location=null, table=test, region=def71812430057c99b7be5281021b22c to restore RIT 2024-11-13T14:13:46,834 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN to state=OFFLINE, location=null, table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3 to restore RIT 2024-11-13T14:13:46,834 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN to state=OFFLINE, location=null, table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c to restore RIT 2024-11-13T14:13:46,835 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256835 2024-11-13T14:13:46,835 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,835 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,836 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,837 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,838 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,838 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,840 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,841 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,842 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,843 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,844 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,845 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:46,845 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507286845 2024-11-13T14:13:46,845 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507346845 2024-11-13T14:13:46,845 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-13T14:13:46,845 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,845 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,845 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,845 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,845 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,845 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,846 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,846 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,847 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,850 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,851 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,852 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,853 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,855 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,856 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,857 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,857 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,857 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,857 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,857 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure ee5b69e1e01798b1055c61a7885f87b4, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,858 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-13T14:13:46,859 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,859 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,859 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,869 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,869 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,869 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,869 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,869 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,869 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,870 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,870 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,870 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,870 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN to state=OFFLINE, location=null, table=test, region=def71812430057c99b7be5281021b22c to restore RIT 2024-11-13T14:13:46,870 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN to state=OFFLINE, location=null, table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3 to restore RIT 2024-11-13T14:13:46,870 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN to state=OFFLINE, location=null, table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c to restore RIT 2024-11-13T14:13:46,870 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256870 2024-11-13T14:13:46,871 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,871 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,872 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,873 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,873 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,873 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,881 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:46,881 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507286881 2024-11-13T14:13:46,881 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507346881 2024-11-13T14:13:46,881 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-13T14:13:46,882 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,882 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,882 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,882 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,882 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,883 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,883 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,886 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,887 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,890 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,891 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,891 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,892 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,892 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,892 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,892 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,892 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,892 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,892 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,892 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,892 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,892 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,892 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,892 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,892 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,892 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN 2024-11-13T14:13:46,892 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-13T14:13:46,893 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,894 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,894 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,907 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,907 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,907 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,908 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,908 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,908 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,908 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,908 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,908 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN to state=OFFLINE, location=null, table=test, region=def71812430057c99b7be5281021b22c to restore RIT 2024-11-13T14:13:46,908 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN to state=OFFLINE, location=null, table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3 to restore RIT 2024-11-13T14:13:46,908 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c, ASSIGN to state=OFFLINE, location=null, table=test, region=eb1bb70b6e4e99eeece01c54dd8a665c to restore RIT 2024-11-13T14:13:46,908 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256908 2024-11-13T14:13:46,908 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,909 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,910 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,911 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,911 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,911 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,911 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,915 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,916 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,917 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,918 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,918 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:46,918 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507286918 2024-11-13T14:13:46,918 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507346918 2024-11-13T14:13:46,918 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-13T14:13:46,918 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,918 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,918 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,918 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,918 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,918 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,918 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,918 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,919 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,921 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,922 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,923 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,925 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,925 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,926 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,926 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN 2024-11-13T14:13:46,926 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-13T14:13:46,927 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,927 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,927 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,937 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,938 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,938 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,938 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,939 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,939 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,939 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,940 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,940 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,940 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,940 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN to state=OFFLINE, location=null, table=test, region=def71812430057c99b7be5281021b22c to restore RIT 2024-11-13T14:13:46,940 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3, ASSIGN to state=OFFLINE, location=null, table=test, region=edbfee0cdbcdc3896eb3a9b618149bc3 to restore RIT 2024-11-13T14:13:46,940 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256940 2024-11-13T14:13:46,940 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,940 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,941 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,943 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,943 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,943 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,945 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,946 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,947 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,948 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,948 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:46,948 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507286948 2024-11-13T14:13:46,948 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507346948 2024-11-13T14:13:46,948 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-13T14:13:46,948 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,948 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,949 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,949 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,949 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,949 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,949 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,949 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,950 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,953 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,954 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,955 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,955 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN 2024-11-13T14:13:46,955 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T14:13:46,956 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,956 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,956 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,963 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,963 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,964 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,965 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=def71812430057c99b7be5281021b22c, ASSIGN to state=OFFLINE, location=null, table=test, region=def71812430057c99b7be5281021b22c to restore RIT 2024-11-13T14:13:46,965 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256965 2024-11-13T14:13:46,965 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,965 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,967 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,968 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,968 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,968 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,968 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,970 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,971 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,972 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,973 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,973 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:46,973 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507286973 2024-11-13T14:13:46,973 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507346973 2024-11-13T14:13:46,973 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-13T14:13:46,973 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:46,973 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,973 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:46,973 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:46,973 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:46,973 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:46,973 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:46,974 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:46,974 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:46,976 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:46,976 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,976 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,976 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,976 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,976 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,976 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,976 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,976 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:46,976 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:46,977 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:46,977 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:46,977 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:46,978 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:46,978 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,978 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,978 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:46,978 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:46,978 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,978 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,979 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,979 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,979 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,979 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,979 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:46,979 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN 2024-11-13T14:13:46,979 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T14:13:46,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:46,980 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:46,981 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:46,991 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,991 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:46,991 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,991 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,991 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:46,992 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:46,992 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:46,992 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:46,992 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:46,992 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2027122fd77a73d52ce4d3368dd23e0a, ASSIGN to state=OFFLINE, location=null, table=test, region=2027122fd77a73d52ce4d3368dd23e0a to restore RIT 2024-11-13T14:13:46,993 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507256993 2024-11-13T14:13:46,993 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:46,993 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,994 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:46,995 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:46,995 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:46,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:46,995 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:46,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,998 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:46,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:46,999 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,000 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,001 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,001 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,001 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287001 2024-11-13T14:13:47,002 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347002 2024-11-13T14:13:47,002 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-13T14:13:47,002 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:47,002 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,002 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,002 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,002 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:47,002 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:47,002 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:47,003 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:47,003 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,006 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:47,007 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:47,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:47,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:47,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:47,009 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,009 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN 2024-11-13T14:13:47,009 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-13T14:13:47,010 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:47,011 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:47,011 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:47,014 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,022 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,023 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,023 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,023 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,023 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:47,023 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:47,023 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:47,023 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=dd3915f2bd1ed9356a65d71ba154c88b, ASSIGN to state=OFFLINE, location=null, table=test, region=dd3915f2bd1ed9356a65d71ba154c88b to restore RIT 2024-11-13T14:13:47,024 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507257024 2024-11-13T14:13:47,024 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:47,024 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,025 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:47,026 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:47,026 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:47,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,026 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,029 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,030 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,031 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,032 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,033 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,033 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287033 2024-11-13T14:13:47,033 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347033 2024-11-13T14:13:47,033 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-13T14:13:47,033 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:47,033 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,033 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,033 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,033 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:47,033 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:47,033 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:47,034 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:47,034 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:47,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:47,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:47,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,037 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,037 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:47,038 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:47,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:47,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,039 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,040 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,040 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:47,040 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,040 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN 2024-11-13T14:13:47,040 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T14:13:47,041 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,041 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:47,042 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:47,042 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:47,054 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,054 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,055 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:47,055 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:47,055 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ef7ba06bec150589d1c840bbd1b13d9b, ASSIGN to state=OFFLINE, location=null, table=test, region=ef7ba06bec150589d1c840bbd1b13d9b to restore RIT 2024-11-13T14:13:47,055 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507257055 2024-11-13T14:13:47,055 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:47,055 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,057 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:47,058 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:47,058 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:47,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,058 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,060 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,061 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,062 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,062 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,063 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287063 2024-11-13T14:13:47,063 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347063 2024-11-13T14:13:47,063 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-13T14:13:47,063 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:47,063 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,063 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,063 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,063 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:47,063 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:47,063 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:47,063 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:47,064 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:47,065 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:47,065 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,065 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,065 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,065 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,065 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,065 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:47,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,066 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:47,066 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,067 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:47,068 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,068 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN 2024-11-13T14:13:47,068 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-13T14:13:47,068 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:47,068 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:47,069 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:47,083 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T14:13:47,106 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,106 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,106 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,106 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,106 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:47,107 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ee5b69e1e01798b1055c61a7885f87b4, ASSIGN to state=OFFLINE, location=null, table=test, region=ee5b69e1e01798b1055c61a7885f87b4 to restore RIT 2024-11-13T14:13:47,107 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507257107 2024-11-13T14:13:47,107 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:47,107 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,108 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:47,109 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:47,109 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:47,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,110 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,112 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,114 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,115 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,115 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,115 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287115 2024-11-13T14:13:47,115 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347115 2024-11-13T14:13:47,115 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-13T14:13:47,115 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:47,115 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,116 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,116 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,116 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:47,116 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:47,116 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:47,116 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:47,117 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:47,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:47,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:47,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,119 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,120 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:47,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:47,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:47,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,122 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:47,122 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,122 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,123 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,123 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,123 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,123 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,123 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,123 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN 2024-11-13T14:13:47,123 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T14:13:47,124 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:47,124 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:47,124 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:47,124 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36813 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,142 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,142 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,143 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,143 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,143 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,144 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=1d3783ebc90fc59392fa002c53f7c56f, ASSIGN to state=OFFLINE, location=null, table=test, region=1d3783ebc90fc59392fa002c53f7c56f to restore RIT 2024-11-13T14:13:47,144 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507257144 2024-11-13T14:13:47,144 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:47,144 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,146 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:47,147 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:47,147 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,147 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,148 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,150 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,151 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,152 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,152 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287152 2024-11-13T14:13:47,152 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347152 2024-11-13T14:13:47,152 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-13T14:13:47,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:47,152 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,152 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,152 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,152 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:47,152 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:47,153 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:47,153 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:47,153 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:47,155 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-13T14:13:47,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:47,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:47,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,155 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,155 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:47,156 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,157 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,157 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-13T14:13:47,158 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:47,158 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:47,158 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:47,170 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,170 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,170 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,170 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,170 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,171 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507257171 2024-11-13T14:13:47,171 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:47,171 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,172 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:47,173 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:47,174 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,174 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,176 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,177 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,178 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,178 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,178 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287178 2024-11-13T14:13:47,178 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347178 2024-11-13T14:13:47,178 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-13T14:13:47,179 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:47,179 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,179 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,179 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,179 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:47,179 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:47,179 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:47,179 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:47,180 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:47,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:47,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:47,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:47,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:47,182 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:47,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,183 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,183 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,183 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,183 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,183 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,183 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.1731507221090, worker=eb1966fd6578,41771,1731507219528 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,183 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-13T14:13:47,184 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:47,184 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:47,184 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:47,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,195 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,195 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,196 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507257196 2024-11-13T14:13:47,196 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:47,196 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,197 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:47,198 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:47,199 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,199 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,201 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,202 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,203 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,203 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287203 2024-11-13T14:13:47,203 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347203 2024-11-13T14:13:47,203 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-13T14:13:47,203 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:47,203 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,203 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,203 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,204 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:47,204 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:47,204 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:47,204 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:47,205 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:47,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:47,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,206 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-13T14:13:47,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:47,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:47,207 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:47,208 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,208 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,208 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,208 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-13T14:13:47,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:47,209 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:47,209 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:47,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,216 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,216 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,216 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,216 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,217 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507257217 2024-11-13T14:13:47,217 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:47,217 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,219 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:47,220 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:47,220 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:47,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,220 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,225 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287225 2024-11-13T14:13:47,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347225 2024-11-13T14:13:47,225 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-13T14:13:47,225 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:47,225 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,226 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,226 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,226 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:47,226 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:47,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:47,226 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:47,227 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:47,228 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:47,228 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,229 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,229 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-13T14:13:47,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:47,229 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:47,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T14:13:47,230 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,230 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,230 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-13T14:13:47,231 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:47,231 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:47,231 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:47,243 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,244 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,244 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,244 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,245 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507257245 2024-11-13T14:13:47,245 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:47,245 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,247 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:47,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:47,249 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,249 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,251 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,252 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,254 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,254 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287254 2024-11-13T14:13:47,254 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347254 2024-11-13T14:13:47,254 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-13T14:13:47,254 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:47,254 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,254 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,254 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,254 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:47,254 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:47,254 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:47,255 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:47,255 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:47,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:47,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,257 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-13T14:13:47,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:47,258 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:47,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,259 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,259 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,259 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-13T14:13:47,260 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:47,260 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:47,260 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:47,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,269 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,269 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,269 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,269 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,269 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,270 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507257270 2024-11-13T14:13:47,270 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:47,270 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,272 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:47,273 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:47,273 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,273 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,273 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,277 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,278 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,279 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287278 2024-11-13T14:13:47,279 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347279 2024-11-13T14:13:47,279 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-13T14:13:47,279 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:47,279 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,279 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,279 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,279 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:47,279 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:47,279 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:47,280 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:47,281 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:47,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:47,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-13T14:13:47,283 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:47,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:47,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,284 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,284 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure eb1966fd6578%2C38145%2C1731507219374.meta.1731507221537.meta, worker=eb1966fd6578,42135,1731507219475 didn't hold the lock before restarting, skip acquiring lock. 2024-11-13T14:13:47,284 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-13T14:13:47,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:47,285 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:47,285 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:47,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,296 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,296 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,296 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,297 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507257297 2024-11-13T14:13:47,297 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:47,297 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,298 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:47,299 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-13T14:13:47,299 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,299 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,299 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,302 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,303 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,304 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,305 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,305 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,305 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287305 2024-11-13T14:13:47,305 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347305 2024-11-13T14:13:47,305 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-13T14:13:47,305 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-13T14:13:47,305 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,305 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,306 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,306 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-13T14:13:47,306 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-13T14:13:47,306 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-13T14:13:47,306 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T14:13:47,307 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-13T14:13:47,309 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-13T14:13:47,309 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T14:13:47,309 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-13T14:13:47,310 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true 2024-11-13T14:13:47,310 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-13T14:13:47,310 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-13T14:13:47,311 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=61 2024-11-13T14:13:47,311 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=eb1966fd6578,42135,1731507219475, table=hbase:meta, region=1588230740 2024-11-13T14:13:47,311 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as eb1966fd6578,42135,1731507219475, state=OPEN 2024-11-13T14:13:47,321 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,321 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T14:13:47,321 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,321 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,322 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T14:13:47,323 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731507257322 2024-11-13T14:13:47,323 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T14:13:47,323 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-13T14:13:47,324 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-13T14:13:47,325 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=73, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure dd3915f2bd1ed9356a65d71ba154c88b, server=eb1966fd6578,42135,1731507219475:java.lang.RuntimeException: inject code bug; ServerCrashProcedure eb1966fd6578,38145,1731507219374, splitWal=true, meta=true exec-time=2.8160 sec 2024-11-13T14:13:47,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b2697dde157dfed90cc8595ff82dc081, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d2b53404c106b2f9960c2de982a64c06, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,328 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0e888576e09ecc86a3b2198a9e11c7b4, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bc9bead55355bc7d07a869ba01fa7404, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=eb1bb70b6e4e99eeece01c54dd8a665c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=17aeba8ae6d5f57b63d67825a02878d1, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=def71812430057c99b7be5281021b22c, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=668c3a8f00f6512df533713b2c2d46cb, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dbe405b0820b2bbf65d6f5d084e654ed, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ef7ba06bec150589d1c840bbd1b13d9b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=409ba30f82abbec05bb6c84a023d1e9a, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=cb7d9e65213673705aa2d05e9c49ef5f, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,329 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edbfee0cdbcdc3896eb3a9b618149bc3, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=026ea48c246e26b4197a66fa854f5022, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=087abf0ac7c707f0f540d20925d9d683, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1d3783ebc90fc59392fa002c53f7c56f, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1f231052a3b0401ad15e94c5bf55a972, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e501f0ec953c86735bbe2081ac738865, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=edf3600642cf1af12a36dc726f301ef2, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dd3915f2bd1ed9356a65d71ba154c88b, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8ebd6e15e1eb3717f4dd8f1695d2b691, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=24857182eb0da41497c7febd6e0c060d, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,330 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2027122fd77a73d52ce4d3368dd23e0a, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=4db6e838ae3d565bc2af54adcdf3a3c0, regionState=OPEN, lastHost=eb1966fd6578,42135,1731507219475, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9bab6e4180453738e691368cc5fbb429, regionState=OPEN, lastHost=eb1966fd6578,41771,1731507219528, regionLocation=eb1966fd6578,41771,1731507219528, openSeqNum=2 2024-11-13T14:13:47,331 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ee5b69e1e01798b1055c61a7885f87b4, regionState=OPENING, lastHost=eb1966fd6578,38145,1731507219374, regionLocation=eb1966fd6578,42135,1731507219475, openSeqNum=2 2024-11-13T14:13:47,331 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-13T14:13:47,331 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731507287331 2024-11-13T14:13:47,331 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731507347331 2024-11-13T14:13:47,331 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-13T14:13:47,348 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=347 (was 381), OpenFileDescriptor=624 (was 647), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=291 (was 291), ProcessCount=11 (was 11), AvailableMemoryMB=3173 (was 3213) 2024-11-13T14:13:47,348 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T14:13:47,349 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T14:13:47,349 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T14:13:47,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T14:13:47,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T14:13:47,349 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T14:13:47,350 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T14:13:47,350 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1651834634, stopped=false 2024-11-13T14:13:47,350 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=eb1966fd6578,36813,1731507218489 2024-11-13T14:13:47,359 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T14:13:47,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T14:13:47,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T14:13:47,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:47,359 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,359 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:47,360 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:47,360 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T14:13:47,360 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T14:13:47,360 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T14:13:47,360 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eb1966fd6578,42135,1731507219475' ***** 2024-11-13T14:13:47,360 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T14:13:47,360 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'eb1966fd6578,41771,1731507219528' ***** 2024-11-13T14:13:47,360 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T14:13:47,360 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T14:13:47,360 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T14:13:47,361 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T14:13:47,361 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T14:13:47,361 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T14:13:47,361 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T14:13:47,361 INFO [RS:2;eb1966fd6578:41771 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T14:13:47,361 INFO [RS:2;eb1966fd6578:41771 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T14:13:47,361 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T14:13:47,361 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for e501f0ec953c86735bbe2081ac738865 2024-11-13T14:13:47,362 INFO [RS:1;eb1966fd6578:42135 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T14:13:47,362 INFO [RS:1;eb1966fd6578:42135 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for 9bab6e4180453738e691368cc5fbb429 2024-11-13T14:13:47,362 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for 8ebd6e15e1eb3717f4dd8f1695d2b691 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for 2027122fd77a73d52ce4d3368dd23e0a 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for 0e888576e09ecc86a3b2198a9e11c7b4 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for edf3600642cf1af12a36dc726f301ef2 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for dbe405b0820b2bbf65d6f5d084e654ed 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for 087abf0ac7c707f0f540d20925d9d683 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for 17aeba8ae6d5f57b63d67825a02878d1 2024-11-13T14:13:47,362 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for d2b53404c106b2f9960c2de982a64c06 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for cb7d9e65213673705aa2d05e9c49ef5f 2024-11-13T14:13:47,362 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for 1f231052a3b0401ad15e94c5bf55a972 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for 24857182eb0da41497c7febd6e0c060d 2024-11-13T14:13:47,362 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for 409ba30f82abbec05bb6c84a023d1e9a 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for eb1bb70b6e4e99eeece01c54dd8a665c 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:47,362 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for 026ea48c246e26b4197a66fa854f5022 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(3091): Received CLOSE for 1d3783ebc90fc59392fa002c53f7c56f 2024-11-13T14:13:47,362 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:47,362 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for def71812430057c99b7be5281021b22c 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(959): stopping server eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,362 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for 4db6e838ae3d565bc2af54adcdf3a3c0 2024-11-13T14:13:47,362 INFO [RS:2;eb1966fd6578:41771 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T14:13:47,362 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for edbfee0cdbcdc3896eb3a9b618149bc3 2024-11-13T14:13:47,362 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e501f0ec953c86735bbe2081ac738865, disabling compactions & flushes 2024-11-13T14:13:47,363 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for b2697dde157dfed90cc8595ff82dc081 2024-11-13T14:13:47,363 INFO [RS:2;eb1966fd6578:41771 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;eb1966fd6578:41771. 2024-11-13T14:13:47,363 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:47,363 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for dd3915f2bd1ed9356a65d71ba154c88b 2024-11-13T14:13:47,363 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for 668c3a8f00f6512df533713b2c2d46cb 2024-11-13T14:13:47,363 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:47,363 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for bc9bead55355bc7d07a869ba01fa7404 2024-11-13T14:13:47,363 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. after waiting 0 ms 2024-11-13T14:13:47,363 DEBUG [RS:2;eb1966fd6578:41771 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T14:13:47,363 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:47,363 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(959): stopping server eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,363 DEBUG [RS:2;eb1966fd6578:41771 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T14:13:47,363 INFO [RS:1;eb1966fd6578:42135 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T14:13:47,363 INFO [RS:1;eb1966fd6578:42135 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;eb1966fd6578:42135. 2024-11-13T14:13:47,363 DEBUG [RS:1;eb1966fd6578:42135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T14:13:47,363 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-13T14:13:47,363 DEBUG [RS:1;eb1966fd6578:42135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T14:13:47,363 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T14:13:47,363 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T14:13:47,363 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T14:13:47,363 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T14:13:47,364 DEBUG [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(1351): Waiting on 087abf0ac7c707f0f540d20925d9d683, 0e888576e09ecc86a3b2198a9e11c7b4, 17aeba8ae6d5f57b63d67825a02878d1, 1d3783ebc90fc59392fa002c53f7c56f, 2027122fd77a73d52ce4d3368dd23e0a, 24857182eb0da41497c7febd6e0c060d, 9bab6e4180453738e691368cc5fbb429, cb7d9e65213673705aa2d05e9c49ef5f, dbe405b0820b2bbf65d6f5d084e654ed, e501f0ec953c86735bbe2081ac738865, eb1bb70b6e4e99eeece01c54dd8a665c, edf3600642cf1af12a36dc726f301ef2, ef7ba06bec150589d1c840bbd1b13d9b 2024-11-13T14:13:47,364 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-13T14:13:47,364 DEBUG [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(1351): Waiting on 026ea48c246e26b4197a66fa854f5022, 1588230740, 1f231052a3b0401ad15e94c5bf55a972, 409ba30f82abbec05bb6c84a023d1e9a, 4db6e838ae3d565bc2af54adcdf3a3c0, 668c3a8f00f6512df533713b2c2d46cb, 8ebd6e15e1eb3717f4dd8f1695d2b691, b2697dde157dfed90cc8595ff82dc081, bc9bead55355bc7d07a869ba01fa7404, d2b53404c106b2f9960c2de982a64c06, dd3915f2bd1ed9356a65d71ba154c88b, def71812430057c99b7be5281021b22c, edbfee0cdbcdc3896eb3a9b618149bc3, ee5b69e1e01798b1055c61a7885f87b4 2024-11-13T14:13:47,364 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8ebd6e15e1eb3717f4dd8f1695d2b691, disabling compactions & flushes 2024-11-13T14:13:47,364 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:47,364 DEBUG [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T14:13:47,364 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:47,364 INFO [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T14:13:47,364 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. after waiting 0 ms 2024-11-13T14:13:47,364 DEBUG [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T14:13:47,364 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:47,364 DEBUG [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T14:13:47,364 DEBUG [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T14:13:47,365 INFO [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.65 KB heapSize=5.44 KB 2024-11-13T14:13:47,372 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/e501f0ec953c86735bbe2081ac738865/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,373 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/8ebd6e15e1eb3717f4dd8f1695d2b691/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,375 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:47,375 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e501f0ec953c86735bbe2081ac738865: Waiting for close lock at 1731507227362Running coprocessor pre-close hooks at 1731507227362Disabling compacts and flushes for region at 1731507227362Disabling writes for close at 1731507227363 (+1 ms)Writing region close event to WAL at 1731507227363Running coprocessor post-close hooks at 1731507227372 (+9 ms)Closed at 1731507227375 (+3 ms) 2024-11-13T14:13:47,375 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:47,375 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8ebd6e15e1eb3717f4dd8f1695d2b691: Waiting for close lock at 1731507227364Running coprocessor pre-close hooks at 1731507227364Disabling compacts and flushes for region at 1731507227364Disabling writes for close at 1731507227364Writing region close event to WAL at 1731507227365 (+1 ms)Running coprocessor post-close hooks at 1731507227375 (+10 ms)Closed at 1731507227375 2024-11-13T14:13:47,375 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731507221993.e501f0ec953c86735bbe2081ac738865. 2024-11-13T14:13:47,375 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731507221993.8ebd6e15e1eb3717f4dd8f1695d2b691. 2024-11-13T14:13:47,376 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d2b53404c106b2f9960c2de982a64c06, disabling compactions & flushes 2024-11-13T14:13:47,376 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9bab6e4180453738e691368cc5fbb429, disabling compactions & flushes 2024-11-13T14:13:47,376 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:47,376 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:47,376 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:47,376 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:47,376 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. after waiting 0 ms 2024-11-13T14:13:47,376 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. after waiting 0 ms 2024-11-13T14:13:47,376 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:47,376 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:47,378 DEBUG [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/info/9a89d4012d044b3e843a58c139760183 is 138, key is test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c./info:regioninfo/1731507226092/Put/seqid=0 2024-11-13T14:13:47,383 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/d2b53404c106b2f9960c2de982a64c06/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,384 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:47,384 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d2b53404c106b2f9960c2de982a64c06: Waiting for close lock at 1731507227376Running coprocessor pre-close hooks at 1731507227376Disabling compacts and flushes for region at 1731507227376Disabling writes for close at 1731507227376Writing region close event to WAL at 1731507227378 (+2 ms)Running coprocessor post-close hooks at 1731507227384 (+6 ms)Closed at 1731507227384 2024-11-13T14:13:47,385 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731507221993.d2b53404c106b2f9960c2de982a64c06. 2024-11-13T14:13:47,385 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1f231052a3b0401ad15e94c5bf55a972, disabling compactions & flushes 2024-11-13T14:13:47,385 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:47,385 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:47,385 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. after waiting 0 ms 2024-11-13T14:13:47,385 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:47,385 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/9bab6e4180453738e691368cc5fbb429/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,386 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:47,386 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9bab6e4180453738e691368cc5fbb429: Waiting for close lock at 1731507227376Running coprocessor pre-close hooks at 1731507227376Disabling compacts and flushes for region at 1731507227376Disabling writes for close at 1731507227376Writing region close event to WAL at 1731507227378 (+2 ms)Running coprocessor post-close hooks at 1731507227386 (+8 ms)Closed at 1731507227386 2024-11-13T14:13:47,386 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731507221993.9bab6e4180453738e691368cc5fbb429. 2024-11-13T14:13:47,386 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2027122fd77a73d52ce4d3368dd23e0a, disabling compactions & flushes 2024-11-13T14:13:47,386 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:47,386 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:47,387 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. after waiting 0 ms 2024-11-13T14:13:47,387 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:47,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741869_1045 (size=8192) 2024-11-13T14:13:47,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741869_1045 (size=8192) 2024-11-13T14:13:47,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741869_1045 (size=8192) 2024-11-13T14:13:47,393 INFO [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.65 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/info/9a89d4012d044b3e843a58c139760183 2024-11-13T14:13:47,394 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/1f231052a3b0401ad15e94c5bf55a972/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,395 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:47,395 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1f231052a3b0401ad15e94c5bf55a972: Waiting for close lock at 1731507227385Running coprocessor pre-close hooks at 1731507227385Disabling compacts and flushes for region at 1731507227385Disabling writes for close at 1731507227385Writing region close event to WAL at 1731507227388 (+3 ms)Running coprocessor post-close hooks at 1731507227395 (+7 ms)Closed at 1731507227395 2024-11-13T14:13:47,395 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731507221993.1f231052a3b0401ad15e94c5bf55a972. 2024-11-13T14:13:47,396 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 409ba30f82abbec05bb6c84a023d1e9a, disabling compactions & flushes 2024-11-13T14:13:47,396 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:47,396 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:47,396 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. after waiting 0 ms 2024-11-13T14:13:47,396 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:47,397 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/2027122fd77a73d52ce4d3368dd23e0a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,399 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:47,399 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2027122fd77a73d52ce4d3368dd23e0a: Waiting for close lock at 1731507227386Running coprocessor pre-close hooks at 1731507227386Disabling compacts and flushes for region at 1731507227386Disabling writes for close at 1731507227387 (+1 ms)Writing region close event to WAL at 1731507227389 (+2 ms)Running coprocessor post-close hooks at 1731507227399 (+10 ms)Closed at 1731507227399 2024-11-13T14:13:47,399 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731507221993.2027122fd77a73d52ce4d3368dd23e0a. 2024-11-13T14:13:47,400 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0e888576e09ecc86a3b2198a9e11c7b4, disabling compactions & flushes 2024-11-13T14:13:47,400 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:47,400 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:47,401 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/409ba30f82abbec05bb6c84a023d1e9a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,401 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. after waiting 0 ms 2024-11-13T14:13:47,401 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:47,401 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:47,402 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 409ba30f82abbec05bb6c84a023d1e9a: Waiting for close lock at 1731507227396Running coprocessor pre-close hooks at 1731507227396Disabling compacts and flushes for region at 1731507227396Disabling writes for close at 1731507227396Writing region close event to WAL at 1731507227396Running coprocessor post-close hooks at 1731507227401 (+5 ms)Closed at 1731507227401 2024-11-13T14:13:47,402 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731507221993.409ba30f82abbec05bb6c84a023d1e9a. 2024-11-13T14:13:47,402 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 026ea48c246e26b4197a66fa854f5022, disabling compactions & flushes 2024-11-13T14:13:47,402 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:47,402 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:47,402 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. after waiting 0 ms 2024-11-13T14:13:47,402 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:47,404 DEBUG [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/.tmp/info/9a89d4012d044b3e843a58c139760183 as hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/info/9a89d4012d044b3e843a58c139760183 2024-11-13T14:13:47,407 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/026ea48c246e26b4197a66fa854f5022/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,408 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:47,408 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 026ea48c246e26b4197a66fa854f5022: Waiting for close lock at 1731507227402Running coprocessor pre-close hooks at 1731507227402Disabling compacts and flushes for region at 1731507227402Disabling writes for close at 1731507227402Writing region close event to WAL at 1731507227402Running coprocessor post-close hooks at 1731507227407 (+5 ms)Closed at 1731507227407 2024-11-13T14:13:47,408 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731507221993.026ea48c246e26b4197a66fa854f5022. 2024-11-13T14:13:47,408 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ee5b69e1e01798b1055c61a7885f87b4, disabling compactions & flushes 2024-11-13T14:13:47,408 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:47,408 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:47,408 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. after waiting 0 ms 2024-11-13T14:13:47,408 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:47,410 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/0e888576e09ecc86a3b2198a9e11c7b4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,412 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:47,412 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0e888576e09ecc86a3b2198a9e11c7b4: Waiting for close lock at 1731507227400Running coprocessor pre-close hooks at 1731507227400Disabling compacts and flushes for region at 1731507227400Disabling writes for close at 1731507227401 (+1 ms)Writing region close event to WAL at 1731507227402 (+1 ms)Running coprocessor post-close hooks at 1731507227411 (+9 ms)Closed at 1731507227412 (+1 ms) 2024-11-13T14:13:47,412 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731507221993.0e888576e09ecc86a3b2198a9e11c7b4. 2024-11-13T14:13:47,412 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing edf3600642cf1af12a36dc726f301ef2, disabling compactions & flushes 2024-11-13T14:13:47,412 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:47,412 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:47,412 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. after waiting 0 ms 2024-11-13T14:13:47,413 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:47,413 INFO [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/info/9a89d4012d044b3e843a58c139760183, entries=24, sequenceid=71, filesize=8 K 2024-11-13T14:13:47,414 INFO [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=71, compaction requested=false 2024-11-13T14:13:47,415 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ee5b69e1e01798b1055c61a7885f87b4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,415 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:47,416 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ee5b69e1e01798b1055c61a7885f87b4: Waiting for close lock at 1731507227408Running coprocessor pre-close hooks at 1731507227408Disabling compacts and flushes for region at 1731507227408Disabling writes for close at 1731507227408Writing region close event to WAL at 1731507227410 (+2 ms)Running coprocessor post-close hooks at 1731507227415 (+5 ms)Closed at 1731507227415 2024-11-13T14:13:47,416 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731507221993.ee5b69e1e01798b1055c61a7885f87b4. 2024-11-13T14:13:47,416 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing def71812430057c99b7be5281021b22c, disabling compactions & flushes 2024-11-13T14:13:47,416 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:47,416 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:47,416 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731507221993.def71812430057c99b7be5281021b22c. after waiting 0 ms 2024-11-13T14:13:47,416 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:47,419 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/edf3600642cf1af12a36dc726f301ef2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,420 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/def71812430057c99b7be5281021b22c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-13T14:13:47,421 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:47,421 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for edf3600642cf1af12a36dc726f301ef2: Waiting for close lock at 1731507227412Running coprocessor pre-close hooks at 1731507227412Disabling compacts and flushes for region at 1731507227412Disabling writes for close at 1731507227413 (+1 ms)Writing region close event to WAL at 1731507227413Running coprocessor post-close hooks at 1731507227421 (+8 ms)Closed at 1731507227421 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for def71812430057c99b7be5281021b22c: Waiting for close lock at 1731507227416Running coprocessor pre-close hooks at 1731507227416Disabling compacts and flushes for region at 1731507227416Disabling writes for close at 1731507227416Writing region close event to WAL at 1731507227417 (+1 ms)Running coprocessor post-close hooks at 1731507227421 (+4 ms)Closed at 1731507227421 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731507221993.def71812430057c99b7be5281021b22c. 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731507221993.edf3600642cf1af12a36dc726f301ef2. 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 4db6e838ae3d565bc2af54adcdf3a3c0, disabling compactions & flushes 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dbe405b0820b2bbf65d6f5d084e654ed, disabling compactions & flushes 2024-11-13T14:13:47,421 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:47,421 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:47,421 INFO [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. after waiting 0 ms 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. after waiting 0 ms 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:47,421 DEBUG [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731507227364Running coprocessor pre-close hooks at 1731507227364Disabling compacts and flushes for region at 1731507227364Disabling writes for close at 1731507227364Obtaining lock to block concurrent updates at 1731507227365 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731507227365Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2717, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731507227370 (+5 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731507227371 (+1 ms)Flushing 1588230740/info: creating writer at 1731507227372 (+1 ms)Flushing 1588230740/info: appending metadata at 1731507227378 (+6 ms)Flushing 1588230740/info: closing flushed file at 1731507227378Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ca63204: reopening flushed file at 1731507227402 (+24 ms)Finished flush of dataSize ~2.65 KB/2717, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 50ms, sequenceid=71, compaction requested=false at 1731507227414 (+12 ms)Writing region close event to WAL at 1731507227416 (+2 ms)Running coprocessor post-close hooks at 1731507227421 (+5 ms)Closed at 1731507227421 2024-11-13T14:13:47,422 DEBUG [RS_CLOSE_META-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T14:13:47,426 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/dbe405b0820b2bbf65d6f5d084e654ed/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,426 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/4db6e838ae3d565bc2af54adcdf3a3c0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,426 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:47,426 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dbe405b0820b2bbf65d6f5d084e654ed: Waiting for close lock at 1731507227421Running coprocessor pre-close hooks at 1731507227421Disabling compacts and flushes for region at 1731507227421Disabling writes for close at 1731507227421Writing region close event to WAL at 1731507227422 (+1 ms)Running coprocessor post-close hooks at 1731507227426 (+4 ms)Closed at 1731507227426 2024-11-13T14:13:47,426 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731507221993.dbe405b0820b2bbf65d6f5d084e654ed. 2024-11-13T14:13:47,427 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 087abf0ac7c707f0f540d20925d9d683, disabling compactions & flushes 2024-11-13T14:13:47,427 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:47,427 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:47,427 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. after waiting 0 ms 2024-11-13T14:13:47,427 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:47,427 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:47,427 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 4db6e838ae3d565bc2af54adcdf3a3c0: Waiting for close lock at 1731507227421Running coprocessor pre-close hooks at 1731507227421Disabling compacts and flushes for region at 1731507227421Disabling writes for close at 1731507227421Writing region close event to WAL at 1731507227422 (+1 ms)Running coprocessor post-close hooks at 1731507227427 (+5 ms)Closed at 1731507227427 2024-11-13T14:13:47,427 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731507221993.4db6e838ae3d565bc2af54adcdf3a3c0. 2024-11-13T14:13:47,427 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing edbfee0cdbcdc3896eb3a9b618149bc3, disabling compactions & flushes 2024-11-13T14:13:47,427 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:47,427 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:47,427 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. after waiting 0 ms 2024-11-13T14:13:47,427 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:47,431 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/087abf0ac7c707f0f540d20925d9d683/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,431 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/edbfee0cdbcdc3896eb3a9b618149bc3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,432 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:47,432 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for edbfee0cdbcdc3896eb3a9b618149bc3: Waiting for close lock at 1731507227427Running coprocessor pre-close hooks at 1731507227427Disabling compacts and flushes for region at 1731507227427Disabling writes for close at 1731507227427Writing region close event to WAL at 1731507227428 (+1 ms)Running coprocessor post-close hooks at 1731507227432 (+4 ms)Closed at 1731507227432 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 087abf0ac7c707f0f540d20925d9d683: Waiting for close lock at 1731507227427Running coprocessor pre-close hooks at 1731507227427Disabling compacts and flushes for region at 1731507227427Disabling writes for close at 1731507227427Writing region close event to WAL at 1731507227427Running coprocessor post-close hooks at 1731507227432 (+5 ms)Closed at 1731507227432 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731507221993.edbfee0cdbcdc3896eb3a9b618149bc3. 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731507221993.087abf0ac7c707f0f540d20925d9d683. 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b2697dde157dfed90cc8595ff82dc081, disabling compactions & flushes 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 17aeba8ae6d5f57b63d67825a02878d1, disabling compactions & flushes 2024-11-13T14:13:47,432 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:47,432 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. after waiting 0 ms 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. after waiting 0 ms 2024-11-13T14:13:47,432 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:47,435 INFO [regionserver/eb1966fd6578:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T14:13:47,437 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/17aeba8ae6d5f57b63d67825a02878d1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,437 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/b2697dde157dfed90cc8595ff82dc081/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,438 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b2697dde157dfed90cc8595ff82dc081: Waiting for close lock at 1731507227432Running coprocessor pre-close hooks at 1731507227432Disabling compacts and flushes for region at 1731507227432Disabling writes for close at 1731507227432Writing region close event to WAL at 1731507227433 (+1 ms)Running coprocessor post-close hooks at 1731507227437 (+4 ms)Closed at 1731507227438 (+1 ms) 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731507221993.b2697dde157dfed90cc8595ff82dc081. 2024-11-13T14:13:47,438 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 17aeba8ae6d5f57b63d67825a02878d1: Waiting for close lock at 1731507227432Running coprocessor pre-close hooks at 1731507227432Disabling compacts and flushes for region at 1731507227432Disabling writes for close at 1731507227432Writing region close event to WAL at 1731507227433 (+1 ms)Running coprocessor post-close hooks at 1731507227438 (+5 ms)Closed at 1731507227438 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dd3915f2bd1ed9356a65d71ba154c88b, disabling compactions & flushes 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731507221993.17aeba8ae6d5f57b63d67825a02878d1. 2024-11-13T14:13:47,438 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. after waiting 0 ms 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing cb7d9e65213673705aa2d05e9c49ef5f, disabling compactions & flushes 2024-11-13T14:13:47,438 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. after waiting 0 ms 2024-11-13T14:13:47,438 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:47,442 INFO [regionserver/eb1966fd6578:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T14:13:47,442 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/dd3915f2bd1ed9356a65d71ba154c88b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,442 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/cb7d9e65213673705aa2d05e9c49ef5f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,443 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:47,443 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:47,443 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dd3915f2bd1ed9356a65d71ba154c88b: Waiting for close lock at 1731507227438Running coprocessor pre-close hooks at 1731507227438Disabling compacts and flushes for region at 1731507227438Disabling writes for close at 1731507227438Writing region close event to WAL at 1731507227438Running coprocessor post-close hooks at 1731507227443 (+5 ms)Closed at 1731507227443 2024-11-13T14:13:47,443 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for cb7d9e65213673705aa2d05e9c49ef5f: Waiting for close lock at 1731507227438Running coprocessor pre-close hooks at 1731507227438Disabling compacts and flushes for region at 1731507227438Disabling writes for close at 1731507227438Writing region close event to WAL at 1731507227438Running coprocessor post-close hooks at 1731507227443 (+5 ms)Closed at 1731507227443 2024-11-13T14:13:47,443 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731507221993.dd3915f2bd1ed9356a65d71ba154c88b. 2024-11-13T14:13:47,443 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731507221993.cb7d9e65213673705aa2d05e9c49ef5f. 2024-11-13T14:13:47,443 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 668c3a8f00f6512df533713b2c2d46cb, disabling compactions & flushes 2024-11-13T14:13:47,443 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 24857182eb0da41497c7febd6e0c060d, disabling compactions & flushes 2024-11-13T14:13:47,444 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:47,444 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:47,444 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:47,444 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:47,444 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. after waiting 0 ms 2024-11-13T14:13:47,444 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. after waiting 0 ms 2024-11-13T14:13:47,444 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:47,444 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:47,448 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/668c3a8f00f6512df533713b2c2d46cb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,448 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/24857182eb0da41497c7febd6e0c060d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,449 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 668c3a8f00f6512df533713b2c2d46cb: Waiting for close lock at 1731507227443Running coprocessor pre-close hooks at 1731507227443Disabling compacts and flushes for region at 1731507227443Disabling writes for close at 1731507227444 (+1 ms)Writing region close event to WAL at 1731507227444Running coprocessor post-close hooks at 1731507227449 (+5 ms)Closed at 1731507227449 2024-11-13T14:13:47,449 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731507221993.668c3a8f00f6512df533713b2c2d46cb. 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 24857182eb0da41497c7febd6e0c060d: Waiting for close lock at 1731507227443Running coprocessor pre-close hooks at 1731507227443Disabling compacts and flushes for region at 1731507227443Disabling writes for close at 1731507227444 (+1 ms)Writing region close event to WAL at 1731507227444Running coprocessor post-close hooks at 1731507227449 (+5 ms)Closed at 1731507227449 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731507221993.24857182eb0da41497c7febd6e0c060d. 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bc9bead55355bc7d07a869ba01fa7404, disabling compactions & flushes 2024-11-13T14:13:47,449 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. after waiting 0 ms 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing eb1bb70b6e4e99eeece01c54dd8a665c, disabling compactions & flushes 2024-11-13T14:13:47,449 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. after waiting 0 ms 2024-11-13T14:13:47,449 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:47,457 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/eb1bb70b6e4e99eeece01c54dd8a665c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,457 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/bc9bead55355bc7d07a869ba01fa7404/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,457 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:47,457 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bc9bead55355bc7d07a869ba01fa7404: Waiting for close lock at 1731507227449Running coprocessor pre-close hooks at 1731507227449Disabling compacts and flushes for region at 1731507227449Disabling writes for close at 1731507227449Writing region close event to WAL at 1731507227450 (+1 ms)Running coprocessor post-close hooks at 1731507227457 (+7 ms)Closed at 1731507227457 2024-11-13T14:13:47,458 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731507221993.bc9bead55355bc7d07a869ba01fa7404. 2024-11-13T14:13:47,458 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:47,458 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for eb1bb70b6e4e99eeece01c54dd8a665c: Waiting for close lock at 1731507227449Running coprocessor pre-close hooks at 1731507227449Disabling compacts and flushes for region at 1731507227449Disabling writes for close at 1731507227449Writing region close event to WAL at 1731507227450 (+1 ms)Running coprocessor post-close hooks at 1731507227458 (+8 ms)Closed at 1731507227458 2024-11-13T14:13:47,458 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731507221993.eb1bb70b6e4e99eeece01c54dd8a665c. 2024-11-13T14:13:47,458 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ef7ba06bec150589d1c840bbd1b13d9b, disabling compactions & flushes 2024-11-13T14:13:47,458 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:47,458 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:47,458 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. after waiting 0 ms 2024-11-13T14:13:47,458 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:47,463 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/ef7ba06bec150589d1c840bbd1b13d9b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,464 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:47,464 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ef7ba06bec150589d1c840bbd1b13d9b: Waiting for close lock at 1731507227458Running coprocessor pre-close hooks at 1731507227458Disabling compacts and flushes for region at 1731507227458Disabling writes for close at 1731507227458Writing region close event to WAL at 1731507227459 (+1 ms)Running coprocessor post-close hooks at 1731507227464 (+5 ms)Closed at 1731507227464 2024-11-13T14:13:47,464 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731507221993.ef7ba06bec150589d1c840bbd1b13d9b. 2024-11-13T14:13:47,464 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1d3783ebc90fc59392fa002c53f7c56f, disabling compactions & flushes 2024-11-13T14:13:47,465 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:47,465 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:47,465 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. after waiting 0 ms 2024-11-13T14:13:47,465 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:47,469 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/data/default/test/1d3783ebc90fc59392fa002c53f7c56f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-13T14:13:47,470 INFO [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:47,470 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1d3783ebc90fc59392fa002c53f7c56f: Waiting for close lock at 1731507227464Running coprocessor pre-close hooks at 1731507227464Disabling compacts and flushes for region at 1731507227464Disabling writes for close at 1731507227465 (+1 ms)Writing region close event to WAL at 1731507227465Running coprocessor post-close hooks at 1731507227470 (+5 ms)Closed at 1731507227470 2024-11-13T14:13:47,470 DEBUG [RS_CLOSE_REGION-regionserver/eb1966fd6578:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731507221993.1d3783ebc90fc59392fa002c53f7c56f. 2024-11-13T14:13:47,564 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(976): stopping server eb1966fd6578,41771,1731507219528; all regions closed. 2024-11-13T14:13:47,564 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(976): stopping server eb1966fd6578,42135,1731507219475; all regions closed. 2024-11-13T14:13:47,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741865_1041 (size=4201) 2024-11-13T14:13:47,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741865_1041 (size=4201) 2024-11-13T14:13:47,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741865_1041 (size=4201) 2024-11-13T14:13:47,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741833_1009 (size=7074) 2024-11-13T14:13:47,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741833_1009 (size=7074) 2024-11-13T14:13:47,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741833_1009 (size=7074) 2024-11-13T14:13:47,573 DEBUG [RS:2;eb1966fd6578:41771 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/oldWALs 2024-11-13T14:13:47,574 INFO [RS:2;eb1966fd6578:41771 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL eb1966fd6578%2C41771%2C1731507219528:(num 1731507221090) 2024-11-13T14:13:47,574 DEBUG [RS:2;eb1966fd6578:41771 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T14:13:47,574 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T14:13:47,574 INFO [RS:2;eb1966fd6578:41771 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T14:13:47,574 DEBUG [RS:1;eb1966fd6578:42135 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/oldWALs 2024-11-13T14:13:47,574 INFO [RS:1;eb1966fd6578:42135 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL eb1966fd6578%2C42135%2C1731507219475.meta:.meta(num 1731507225346) 2024-11-13T14:13:47,574 INFO [RS:2;eb1966fd6578:41771 {}] hbase.ChoreService(370): Chore service for: regionserver/eb1966fd6578:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T14:13:47,574 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T14:13:47,575 INFO [regionserver/eb1966fd6578:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T14:13:47,575 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T14:13:47,575 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T14:13:47,575 INFO [RS:2;eb1966fd6578:41771 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T14:13:47,575 INFO [RS:2;eb1966fd6578:41771 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41771 2024-11-13T14:13:47,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741835_1011 (size=7064) 2024-11-13T14:13:47,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741835_1011 (size=7064) 2024-11-13T14:13:47,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741835_1011 (size=7064) 2024-11-13T14:13:47,580 DEBUG [RS:1;eb1966fd6578:42135 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/oldWALs 2024-11-13T14:13:47,580 INFO [RS:1;eb1966fd6578:42135 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL eb1966fd6578%2C42135%2C1731507219475:(num 1731507221090) 2024-11-13T14:13:47,580 DEBUG [RS:1;eb1966fd6578:42135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T14:13:47,580 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T14:13:47,580 INFO [RS:1;eb1966fd6578:42135 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T14:13:47,580 INFO [RS:1;eb1966fd6578:42135 {}] hbase.ChoreService(370): Chore service for: regionserver/eb1966fd6578:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T14:13:47,580 INFO [RS:1;eb1966fd6578:42135 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T14:13:47,580 INFO [regionserver/eb1966fd6578:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T14:13:47,581 INFO [RS:1;eb1966fd6578:42135 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:42135 2024-11-13T14:13:47,584 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/eb1966fd6578,41771,1731507219528 2024-11-13T14:13:47,584 INFO [RS:2;eb1966fd6578:41771 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T14:13:47,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T14:13:47,595 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/eb1966fd6578,42135,1731507219475 2024-11-13T14:13:47,595 INFO [RS:1;eb1966fd6578:42135 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T14:13:47,605 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eb1966fd6578,41771,1731507219528] 2024-11-13T14:13:47,626 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/eb1966fd6578,41771,1731507219528 already deleted, retry=false 2024-11-13T14:13:47,627 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; eb1966fd6578,41771,1731507219528 expired; onlineServers=1 2024-11-13T14:13:47,627 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [eb1966fd6578,42135,1731507219475] 2024-11-13T14:13:47,637 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/eb1966fd6578,42135,1731507219475 already deleted, retry=false 2024-11-13T14:13:47,637 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; eb1966fd6578,42135,1731507219475 expired; onlineServers=0 2024-11-13T14:13:47,637 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'eb1966fd6578,36813,1731507218489' ***** 2024-11-13T14:13:47,637 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T14:13:47,637 INFO [M:0;eb1966fd6578:36813 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T14:13:47,637 INFO [M:0;eb1966fd6578:36813 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T14:13:47,637 DEBUG [M:0;eb1966fd6578:36813 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T14:13:47,638 DEBUG [M:0;eb1966fd6578:36813 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T14:13:47,638 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T14:13:47,638 DEBUG [master/eb1966fd6578:0:becomeActiveMaster-HFileCleaner.small.0-1731507220709 {}] cleaner.HFileCleaner(306): Exit Thread[master/eb1966fd6578:0:becomeActiveMaster-HFileCleaner.small.0-1731507220709,5,FailOnTimeoutGroup] 2024-11-13T14:13:47,638 DEBUG [master/eb1966fd6578:0:becomeActiveMaster-HFileCleaner.large.0-1731507220706 {}] cleaner.HFileCleaner(306): Exit Thread[master/eb1966fd6578:0:becomeActiveMaster-HFileCleaner.large.0-1731507220706,5,FailOnTimeoutGroup] 2024-11-13T14:13:47,638 INFO [M:0;eb1966fd6578:36813 {}] hbase.ChoreService(370): Chore service for: master/eb1966fd6578:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T14:13:47,638 INFO [M:0;eb1966fd6578:36813 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T14:13:47,638 DEBUG [M:0;eb1966fd6578:36813 {}] master.HMaster(1795): Stopping service threads 2024-11-13T14:13:47,638 INFO [M:0;eb1966fd6578:36813 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T14:13:47,638 INFO [M:0;eb1966fd6578:36813 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T14:13:47,638 INFO [M:0;eb1966fd6578:36813 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T14:13:47,638 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T14:13:47,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T14:13:47,647 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T14:13:47,647 DEBUG [M:0;eb1966fd6578:36813 {}] zookeeper.ZKUtil(347): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T14:13:47,647 WARN [M:0;eb1966fd6578:36813 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T14:13:47,648 INFO [M:0;eb1966fd6578:36813 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/.lastflushedseqids 2024-11-13T14:13:47,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741870_1046 (size=1832) 2024-11-13T14:13:47,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741870_1046 (size=1832) 2024-11-13T14:13:47,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741870_1046 (size=1832) 2024-11-13T14:13:47,662 INFO [M:0;eb1966fd6578:36813 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T14:13:47,662 INFO [M:0;eb1966fd6578:36813 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T14:13:47,662 DEBUG [M:0;eb1966fd6578:36813 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T14:13:47,662 INFO [M:0;eb1966fd6578:36813 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T14:13:47,662 DEBUG [M:0;eb1966fd6578:36813 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T14:13:47,662 DEBUG [M:0;eb1966fd6578:36813 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T14:13:47,662 DEBUG [M:0;eb1966fd6578:36813 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T14:13:47,662 INFO [M:0;eb1966fd6578:36813 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=250.05 KB heapSize=303.88 KB 2024-11-13T14:13:47,679 DEBUG [M:0;eb1966fd6578:36813 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3da2f91a67c746528263473702e3ff65 is 82, key is hbase:meta,,1/info:regioninfo/1731507225659/Put/seqid=0 2024-11-13T14:13:47,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741871_1047 (size=6071) 2024-11-13T14:13:47,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741871_1047 (size=6071) 2024-11-13T14:13:47,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741871_1047 (size=6071) 2024-11-13T14:13:47,686 INFO [M:0;eb1966fd6578:36813 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3da2f91a67c746528263473702e3ff65 2024-11-13T14:13:47,706 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T14:13:47,706 INFO [RS:2;eb1966fd6578:41771 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T14:13:47,706 DEBUG [pool-75-thread-1-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41771-0x101348b60910003, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T14:13:47,706 INFO [RS:2;eb1966fd6578:41771 {}] regionserver.HRegionServer(1031): Exiting; stopping=eb1966fd6578,41771,1731507219528; zookeeper connection closed. 2024-11-13T14:13:47,706 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5540f925 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5540f925 2024-11-13T14:13:47,709 DEBUG [M:0;eb1966fd6578:36813 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cb9c530acb2e416aaa2f26c9de6d2698 is 2987, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1731507227324/Put/seqid=0 2024-11-13T14:13:47,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741872_1048 (size=12797) 2024-11-13T14:13:47,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741872_1048 (size=12797) 2024-11-13T14:13:47,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741872_1048 (size=12797) 2024-11-13T14:13:47,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T14:13:47,716 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42135-0x101348b60910002, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T14:13:47,716 INFO [RS:1;eb1966fd6578:42135 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T14:13:47,716 INFO [RS:1;eb1966fd6578:42135 {}] regionserver.HRegionServer(1031): Exiting; stopping=eb1966fd6578,42135,1731507219475; zookeeper connection closed. 2024-11-13T14:13:47,716 INFO [M:0;eb1966fd6578:36813 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=248.77 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cb9c530acb2e416aaa2f26c9de6d2698 2024-11-13T14:13:47,717 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@18b389a4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@18b389a4 2024-11-13T14:13:47,717 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T14:13:47,737 DEBUG [M:0;eb1966fd6578:36813 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e6679786999442cb5a6d05b59628e2d is 69, key is eb1966fd6578,41771,1731507219528/rs:state/1731507220802/Put/seqid=0 2024-11-13T14:13:47,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741873_1049 (size=5442) 2024-11-13T14:13:47,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741873_1049 (size=5442) 2024-11-13T14:13:47,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741873_1049 (size=5442) 2024-11-13T14:13:47,745 INFO [M:0;eb1966fd6578:36813 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e6679786999442cb5a6d05b59628e2d 2024-11-13T14:13:47,751 INFO [M:0;eb1966fd6578:36813 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9e6679786999442cb5a6d05b59628e2d 2024-11-13T14:13:47,764 DEBUG [M:0;eb1966fd6578:36813 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a51ab15b9a7143e2adce677e9efc4b07 is 52, key is load_balancer_on/state:d/1731507224344/Put/seqid=0 2024-11-13T14:13:47,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741874_1050 (size=5058) 2024-11-13T14:13:47,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741874_1050 (size=5058) 2024-11-13T14:13:47,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741874_1050 (size=5058) 2024-11-13T14:13:47,771 INFO [M:0;eb1966fd6578:36813 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a51ab15b9a7143e2adce677e9efc4b07 2024-11-13T14:13:47,778 DEBUG [M:0;eb1966fd6578:36813 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/3da2f91a67c746528263473702e3ff65 as hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3da2f91a67c746528263473702e3ff65 2024-11-13T14:13:47,784 INFO [M:0;eb1966fd6578:36813 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/3da2f91a67c746528263473702e3ff65, entries=14, sequenceid=575, filesize=5.9 K 2024-11-13T14:13:47,785 DEBUG [M:0;eb1966fd6578:36813 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/cb9c530acb2e416aaa2f26c9de6d2698 as hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cb9c530acb2e416aaa2f26c9de6d2698 2024-11-13T14:13:47,792 INFO [M:0;eb1966fd6578:36813 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/cb9c530acb2e416aaa2f26c9de6d2698, entries=79, sequenceid=575, filesize=12.5 K 2024-11-13T14:13:47,793 DEBUG [M:0;eb1966fd6578:36813 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e6679786999442cb5a6d05b59628e2d as hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9e6679786999442cb5a6d05b59628e2d 2024-11-13T14:13:47,800 INFO [M:0;eb1966fd6578:36813 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9e6679786999442cb5a6d05b59628e2d 2024-11-13T14:13:47,800 INFO [M:0;eb1966fd6578:36813 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9e6679786999442cb5a6d05b59628e2d, entries=3, sequenceid=575, filesize=5.3 K 2024-11-13T14:13:47,801 DEBUG [M:0;eb1966fd6578:36813 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/a51ab15b9a7143e2adce677e9efc4b07 as hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a51ab15b9a7143e2adce677e9efc4b07 2024-11-13T14:13:47,807 INFO [M:0;eb1966fd6578:36813 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34289/user/jenkins/test-data/92a9c591-4bcd-53fb-192f-c196ffdeae9b/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/a51ab15b9a7143e2adce677e9efc4b07, entries=1, sequenceid=575, filesize=4.9 K 2024-11-13T14:13:47,808 INFO [M:0;eb1966fd6578:36813 {}] regionserver.HRegion(3140): Finished flush of dataSize ~250.05 KB/256048, heapSize ~303.82 KB/311112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 146ms, sequenceid=575, compaction requested=false 2024-11-13T14:13:47,810 INFO [M:0;eb1966fd6578:36813 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T14:13:47,810 DEBUG [M:0;eb1966fd6578:36813 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731507227662Disabling compacts and flushes for region at 1731507227662Disabling writes for close at 1731507227662Obtaining lock to block concurrent updates at 1731507227662Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731507227662Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=256048, getHeapSize=311112, getOffHeapSize=0, getCellsCount=714 at 1731507227663 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731507227663Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731507227663Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731507227678 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731507227678Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731507227691 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731507227708 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731507227708Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731507227723 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731507227737 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731507227737Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731507227751 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731507227764 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731507227764Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bc13300: reopening flushed file at 1731507227777 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e1c87dd: reopening flushed file at 1731507227784 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2de2d8c: reopening flushed file at 1731507227792 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@294f2a70: reopening flushed file at 1731507227800 (+8 ms)Finished flush of dataSize ~250.05 KB/256048, heapSize ~303.82 KB/311112, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 146ms, sequenceid=575, compaction requested=false at 1731507227808 (+8 ms)Writing region close event to WAL at 1731507227810 (+2 ms)Closed at 1731507227810 2024-11-13T14:13:47,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46747 is added to blk_1073741830_1006 (size=294432) 2024-11-13T14:13:47,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:35125 is added to blk_1073741830_1006 (size=294432) 2024-11-13T14:13:47,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33015 is added to blk_1073741830_1006 (size=294432) 2024-11-13T14:13:47,813 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T14:13:47,813 INFO [M:0;eb1966fd6578:36813 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T14:13:47,813 INFO [M:0;eb1966fd6578:36813 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36813 2024-11-13T14:13:47,813 INFO [M:0;eb1966fd6578:36813 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T14:13:47,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T14:13:47,923 INFO [M:0;eb1966fd6578:36813 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T14:13:47,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36813-0x101348b60910000, quorum=127.0.0.1:59966, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T14:13:47,954 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@27d2afd6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T14:13:47,958 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1f351b31{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T14:13:47,958 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T14:13:47,958 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@796aba77{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T14:13:47,958 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48261703{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/hadoop.log.dir/,STOPPED} 2024-11-13T14:13:47,961 WARN [BP-1546972301-172.17.0.2-1731507214000 heartbeating to localhost/127.0.0.1:34289 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T14:13:47,961 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T14:13:47,961 WARN [BP-1546972301-172.17.0.2-1731507214000 heartbeating to localhost/127.0.0.1:34289 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1546972301-172.17.0.2-1731507214000 (Datanode Uuid 253dc82c-cfad-4b53-804e-f665b53be144) service to localhost/127.0.0.1:34289 2024-11-13T14:13:47,961 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T14:13:47,962 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data5/current/BP-1546972301-172.17.0.2-1731507214000 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T14:13:47,962 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data6/current/BP-1546972301-172.17.0.2-1731507214000 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T14:13:47,962 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T14:13:47,964 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1873c69b{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T14:13:47,965 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6ca981a0{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T14:13:47,965 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T14:13:47,965 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2041eb0d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T14:13:47,965 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@9c3cb41{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/hadoop.log.dir/,STOPPED} 2024-11-13T14:13:47,966 WARN [BP-1546972301-172.17.0.2-1731507214000 heartbeating to localhost/127.0.0.1:34289 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T14:13:47,966 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T14:13:47,966 WARN [BP-1546972301-172.17.0.2-1731507214000 heartbeating to localhost/127.0.0.1:34289 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1546972301-172.17.0.2-1731507214000 (Datanode Uuid 1726ac66-1101-4851-b1bf-95c200689e57) service to localhost/127.0.0.1:34289 2024-11-13T14:13:47,966 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T14:13:47,966 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data3/current/BP-1546972301-172.17.0.2-1731507214000 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T14:13:47,967 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data4/current/BP-1546972301-172.17.0.2-1731507214000 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T14:13:47,967 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T14:13:47,969 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@715133de{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T14:13:47,969 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6d8e9708{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T14:13:47,969 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T14:13:47,970 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@70201ae6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T14:13:47,970 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@40b6b75c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/hadoop.log.dir/,STOPPED} 2024-11-13T14:13:47,971 WARN [BP-1546972301-172.17.0.2-1731507214000 heartbeating to localhost/127.0.0.1:34289 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T14:13:47,971 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T14:13:47,971 WARN [BP-1546972301-172.17.0.2-1731507214000 heartbeating to localhost/127.0.0.1:34289 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1546972301-172.17.0.2-1731507214000 (Datanode Uuid c9bb1ff7-beb3-4bf2-b8ea-918fdcc719f3) service to localhost/127.0.0.1:34289 2024-11-13T14:13:47,971 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T14:13:47,972 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data1/current/BP-1546972301-172.17.0.2-1731507214000 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T14:13:47,972 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/cluster_093386a9-be91-c718-16a4-560a1dd2bc7b/data/data2/current/BP-1546972301-172.17.0.2-1731507214000 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T14:13:47,972 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T14:13:47,981 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@376fa8f3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T14:13:47,981 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T14:13:47,982 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T14:13:47,982 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T14:13:47,982 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/b205f3e4-4913-54cf-1321-476b8babf036/hadoop.log.dir/,STOPPED} 2024-11-13T14:13:47,992 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T14:13:48,042 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down