2024-11-21 10:13:14,245 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-21 10:13:14,260 main DEBUG Took 0.012450 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-21 10:13:14,260 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-21 10:13:14,260 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-21 10:13:14,261 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-21 10:13:14,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,269 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-21 10:13:14,280 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,282 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,283 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,283 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,284 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,284 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,285 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,285 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,286 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,286 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,287 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,288 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,288 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,288 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,289 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,290 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,290 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-21 10:13:14,290 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,291 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-21 10:13:14,292 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-21 10:13:14,293 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-21 10:13:14,295 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-21 10:13:14,295 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-21 10:13:14,296 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-21 10:13:14,297 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-21 10:13:14,304 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-21 10:13:14,306 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-21 10:13:14,308 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-21 10:13:14,308 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-21 10:13:14,309 main DEBUG createAppenders(={Console}) 2024-11-21 10:13:14,309 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-21 10:13:14,310 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-21 10:13:14,310 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-21 10:13:14,310 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-21 10:13:14,310 main DEBUG OutputStream closed 2024-11-21 10:13:14,311 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-21 10:13:14,311 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-21 10:13:14,311 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-21 10:13:14,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-21 10:13:14,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-21 10:13:14,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-21 10:13:14,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-21 10:13:14,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-21 10:13:14,384 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-21 10:13:14,384 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-21 10:13:14,384 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-21 10:13:14,385 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-21 10:13:14,385 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-21 10:13:14,385 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-21 10:13:14,385 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-21 10:13:14,386 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-21 10:13:14,386 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-21 10:13:14,386 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-21 10:13:14,387 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-21 10:13:14,387 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-21 10:13:14,388 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-21 10:13:14,390 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21 10:13:14,390 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-21 10:13:14,390 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-21 10:13:14,391 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-21T10:13:14,403 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-21 10:13:14,405 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-21 10:13:14,405 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-21T10:13:14,622 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04 2024-11-21T10:13:14,639 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-21T10:13:14,656 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f, deleteOnExit=true 2024-11-21T10:13:14,657 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-21T10:13:14,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/test.cache.data in system properties and HBase conf 2024-11-21T10:13:14,658 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/hadoop.tmp.dir in system properties and HBase conf 2024-11-21T10:13:14,659 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/hadoop.log.dir in system properties and HBase conf 2024-11-21T10:13:14,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-21T10:13:14,660 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-21T10:13:14,661 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-21T10:13:14,748 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-21T10:13:14,836 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-21T10:13:14,840 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-21T10:13:14,841 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-21T10:13:14,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-21T10:13:14,842 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T10:13:14,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-21T10:13:14,843 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-21T10:13:14,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-21T10:13:14,844 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T10:13:14,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-21T10:13:14,845 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/nfs.dump.dir in system properties and HBase conf 2024-11-21T10:13:14,846 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/java.io.tmpdir in system properties and HBase conf 2024-11-21T10:13:14,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-21T10:13:14,847 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-21T10:13:14,848 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-21T10:13:15,947 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-21T10:13:16,018 INFO [Time-limited test {}] log.Log(170): Logging initialized @2469ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-21T10:13:16,091 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T10:13:16,185 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T10:13:16,216 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T10:13:16,217 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T10:13:16,219 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T10:13:16,253 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T10:13:16,258 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/hadoop.log.dir/,AVAILABLE} 2024-11-21T10:13:16,259 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T10:13:16,500 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/java.io.tmpdir/jetty-localhost-40619-hadoop-hdfs-3_4_1-tests_jar-_-any-15048466538213184945/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T10:13:16,507 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:40619} 2024-11-21T10:13:16,507 INFO [Time-limited test {}] server.Server(415): Started @2959ms 2024-11-21T10:13:17,189 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T10:13:17,196 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T10:13:17,197 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T10:13:17,197 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T10:13:17,197 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T10:13:17,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/hadoop.log.dir/,AVAILABLE} 2024-11-21T10:13:17,198 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T10:13:17,299 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/java.io.tmpdir/jetty-localhost-36343-hadoop-hdfs-3_4_1-tests_jar-_-any-3154344447046744708/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T10:13:17,299 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:36343} 2024-11-21T10:13:17,299 INFO [Time-limited test {}] server.Server(415): Started @3752ms 2024-11-21T10:13:17,365 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T10:13:17,672 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T10:13:17,694 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T10:13:17,727 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T10:13:17,727 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T10:13:17,728 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-21T10:13:17,741 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/hadoop.log.dir/,AVAILABLE} 2024-11-21T10:13:17,742 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T10:13:17,889 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/java.io.tmpdir/jetty-localhost-40295-hadoop-hdfs-3_4_1-tests_jar-_-any-4549547422549648248/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T10:13:17,890 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:40295} 2024-11-21T10:13:17,890 INFO [Time-limited test {}] server.Server(415): Started @4342ms 2024-11-21T10:13:17,893 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T10:13:17,993 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-21T10:13:18,002 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-21T10:13:18,009 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-21T10:13:18,009 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-21T10:13:18,010 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-21T10:13:18,021 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/hadoop.log.dir/,AVAILABLE} 2024-11-21T10:13:18,022 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-21T10:13:18,175 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/java.io.tmpdir/jetty-localhost-34999-hadoop-hdfs-3_4_1-tests_jar-_-any-10538576597462285391/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T10:13:18,176 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:34999} 2024-11-21T10:13:18,177 INFO [Time-limited test {}] server.Server(415): Started @4629ms 2024-11-21T10:13:18,181 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-21T10:13:19,418 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data3/current/BP-1964512914-172.17.0.2-1732183995432/current, will proceed with Du for space computation calculation, 2024-11-21T10:13:19,418 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data1/current/BP-1964512914-172.17.0.2-1732183995432/current, will proceed with Du for space computation calculation, 2024-11-21T10:13:19,418 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data2/current/BP-1964512914-172.17.0.2-1732183995432/current, will proceed with Du for space computation calculation, 2024-11-21T10:13:19,418 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data4/current/BP-1964512914-172.17.0.2-1732183995432/current, will proceed with Du for space computation calculation, 2024-11-21T10:13:19,452 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T10:13:19,452 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T10:13:19,500 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2269e5d4bcd4c0d9 with lease ID 0xbb3f07af5ae3f225: Processing first storage report for DS-ebed3a63-7f40-4df1-bc25-cc54b1ace770 from datanode DatanodeRegistration(127.0.0.1:45815, datanodeUuid=c231f160-e49d-4029-975a-f35ca7bca8bc, infoPort=33461, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432) 2024-11-21T10:13:19,501 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2269e5d4bcd4c0d9 with lease ID 0xbb3f07af5ae3f225: from storage DS-ebed3a63-7f40-4df1-bc25-cc54b1ace770 node DatanodeRegistration(127.0.0.1:45815, datanodeUuid=c231f160-e49d-4029-975a-f35ca7bca8bc, infoPort=33461, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T10:13:19,502 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3089ca4bd76bfdf2 with lease ID 0xbb3f07af5ae3f224: Processing first storage report for DS-5d8f2e7a-cf62-4175-abda-564b3f52e773 from datanode DatanodeRegistration(127.0.0.1:37847, datanodeUuid=908e4323-5680-4328-b07f-2c5ad8591ae9, infoPort=33871, infoSecurePort=0, ipcPort=40245, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432) 2024-11-21T10:13:19,502 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3089ca4bd76bfdf2 with lease ID 0xbb3f07af5ae3f224: from storage DS-5d8f2e7a-cf62-4175-abda-564b3f52e773 node DatanodeRegistration(127.0.0.1:37847, datanodeUuid=908e4323-5680-4328-b07f-2c5ad8591ae9, infoPort=33871, infoSecurePort=0, ipcPort=40245, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T10:13:19,502 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x2269e5d4bcd4c0d9 with lease ID 0xbb3f07af5ae3f225: Processing first storage report for DS-d678af98-1fd4-47f1-bfea-5082e6033657 from datanode DatanodeRegistration(127.0.0.1:45815, datanodeUuid=c231f160-e49d-4029-975a-f35ca7bca8bc, infoPort=33461, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432) 2024-11-21T10:13:19,503 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x2269e5d4bcd4c0d9 with lease ID 0xbb3f07af5ae3f225: from storage DS-d678af98-1fd4-47f1-bfea-5082e6033657 node DatanodeRegistration(127.0.0.1:45815, datanodeUuid=c231f160-e49d-4029-975a-f35ca7bca8bc, infoPort=33461, infoSecurePort=0, ipcPort=39611, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T10:13:19,503 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3089ca4bd76bfdf2 with lease ID 0xbb3f07af5ae3f224: Processing first storage report for DS-d1df3ceb-f1ed-438c-9482-3244c3bea251 from datanode DatanodeRegistration(127.0.0.1:37847, datanodeUuid=908e4323-5680-4328-b07f-2c5ad8591ae9, infoPort=33871, infoSecurePort=0, ipcPort=40245, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432) 2024-11-21T10:13:19,503 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3089ca4bd76bfdf2 with lease ID 0xbb3f07af5ae3f224: from storage DS-d1df3ceb-f1ed-438c-9482-3244c3bea251 node DatanodeRegistration(127.0.0.1:37847, datanodeUuid=908e4323-5680-4328-b07f-2c5ad8591ae9, infoPort=33871, infoSecurePort=0, ipcPort=40245, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T10:13:19,570 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data5/current/BP-1964512914-172.17.0.2-1732183995432/current, will proceed with Du for space computation calculation, 2024-11-21T10:13:19,570 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data6/current/BP-1964512914-172.17.0.2-1732183995432/current, will proceed with Du for space computation calculation, 2024-11-21T10:13:19,590 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-21T10:13:19,594 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad81f244eecd4ea4 with lease ID 0xbb3f07af5ae3f226: Processing first storage report for DS-3d33f7b2-5674-4258-8003-1641935bc33a from datanode DatanodeRegistration(127.0.0.1:39431, datanodeUuid=8bf2038d-9705-4e08-a17d-9c0aeb010e6c, infoPort=44629, infoSecurePort=0, ipcPort=37837, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432) 2024-11-21T10:13:19,594 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad81f244eecd4ea4 with lease ID 0xbb3f07af5ae3f226: from storage DS-3d33f7b2-5674-4258-8003-1641935bc33a node DatanodeRegistration(127.0.0.1:39431, datanodeUuid=8bf2038d-9705-4e08-a17d-9c0aeb010e6c, infoPort=44629, infoSecurePort=0, ipcPort=37837, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-21T10:13:19,594 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xad81f244eecd4ea4 with lease ID 0xbb3f07af5ae3f226: Processing first storage report for DS-5477c7d6-c6ef-462d-b4f2-65bcc2adb68f from datanode DatanodeRegistration(127.0.0.1:39431, datanodeUuid=8bf2038d-9705-4e08-a17d-9c0aeb010e6c, infoPort=44629, infoSecurePort=0, ipcPort=37837, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432) 2024-11-21T10:13:19,594 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xad81f244eecd4ea4 with lease ID 0xbb3f07af5ae3f226: from storage DS-5477c7d6-c6ef-462d-b4f2-65bcc2adb68f node DatanodeRegistration(127.0.0.1:39431, datanodeUuid=8bf2038d-9705-4e08-a17d-9c0aeb010e6c, infoPort=44629, infoSecurePort=0, ipcPort=37837, storageInfo=lv=-57;cid=testClusterID;nsid=1733066663;c=1732183995432), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-21T10:13:19,613 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04 2024-11-21T10:13:19,675 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/zookeeper_0, clientPort=55007, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-21T10:13:19,684 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55007 2024-11-21T10:13:19,696 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T10:13:19,699 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T10:13:19,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741825_1001 (size=7) 2024-11-21T10:13:19,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741825_1001 (size=7) 2024-11-21T10:13:19,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741825_1001 (size=7) 2024-11-21T10:13:20,310 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff with version=8 2024-11-21T10:13:20,311 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/hbase-staging 2024-11-21T10:13:20,394 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-21T10:13:20,626 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2651e482101a:0 server-side Connection retries=45 2024-11-21T10:13:20,634 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:20,634 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:20,638 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T10:13:20,639 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:20,639 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T10:13:20,769 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-21T10:13:20,831 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-21T10:13:20,840 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-21T10:13:20,843 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T10:13:20,866 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2330 (auto-detected) 2024-11-21T10:13:20,867 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-21T10:13:20,884 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:35251 2024-11-21T10:13:20,904 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:35251 connecting to ZooKeeper ensemble=127.0.0.1:55007 2024-11-21T10:13:21,016 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:352510x0, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T10:13:21,019 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:35251-0x1015ce240570000 connected 2024-11-21T10:13:21,116 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T10:13:21,120 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T10:13:21,132 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T10:13:21,136 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff, hbase.cluster.distributed=false 2024-11-21T10:13:21,160 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T10:13:21,165 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35251 2024-11-21T10:13:21,166 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35251 2024-11-21T10:13:21,168 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35251 2024-11-21T10:13:21,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35251 2024-11-21T10:13:21,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35251 2024-11-21T10:13:21,261 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2651e482101a:0 server-side Connection retries=45 2024-11-21T10:13:21,262 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:21,262 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:21,262 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T10:13:21,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:21,263 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T10:13:21,265 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T10:13:21,268 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T10:13:21,269 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36083 2024-11-21T10:13:21,271 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36083 connecting to ZooKeeper ensemble=127.0.0.1:55007 2024-11-21T10:13:21,272 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T10:13:21,274 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T10:13:21,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:360830x0, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T10:13:21,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:360830x0, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T10:13:21,455 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36083-0x1015ce240570001 connected 2024-11-21T10:13:21,459 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T10:13:21,466 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T10:13:21,469 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T10:13:21,473 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T10:13:21,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36083 2024-11-21T10:13:21,474 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36083 2024-11-21T10:13:21,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36083 2024-11-21T10:13:21,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36083 2024-11-21T10:13:21,475 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36083 2024-11-21T10:13:21,491 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2651e482101a:0 server-side Connection retries=45 2024-11-21T10:13:21,491 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:21,492 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:21,492 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T10:13:21,492 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:21,492 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T10:13:21,492 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T10:13:21,493 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T10:13:21,493 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36761 2024-11-21T10:13:21,495 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:36761 connecting to ZooKeeper ensemble=127.0.0.1:55007 2024-11-21T10:13:21,496 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T10:13:21,498 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T10:13:21,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:367610x0, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T10:13:21,526 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:367610x0, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T10:13:21,526 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36761-0x1015ce240570002 connected 2024-11-21T10:13:21,526 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T10:13:21,527 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T10:13:21,528 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T10:13:21,530 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T10:13:21,531 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36761 2024-11-21T10:13:21,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36761 2024-11-21T10:13:21,532 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36761 2024-11-21T10:13:21,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36761 2024-11-21T10:13:21,533 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36761 2024-11-21T10:13:21,551 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2651e482101a:0 server-side Connection retries=45 2024-11-21T10:13:21,552 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:21,552 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:21,552 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-21T10:13:21,552 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-21T10:13:21,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-21T10:13:21,553 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-21T10:13:21,553 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-21T10:13:21,554 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33515 2024-11-21T10:13:21,556 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33515 connecting to ZooKeeper ensemble=127.0.0.1:55007 2024-11-21T10:13:21,557 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T10:13:21,560 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T10:13:21,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335150x0, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-21T10:13:21,574 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:335150x0, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T10:13:21,574 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33515-0x1015ce240570003 connected 2024-11-21T10:13:21,575 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-21T10:13:21,575 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-21T10:13:21,576 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-21T10:13:21,578 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-21T10:13:21,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33515 2024-11-21T10:13:21,579 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33515 2024-11-21T10:13:21,580 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33515 2024-11-21T10:13:21,581 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33515 2024-11-21T10:13:21,582 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33515 2024-11-21T10:13:21,597 DEBUG [M:0;2651e482101a:35251 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2651e482101a:35251 2024-11-21T10:13:21,598 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2651e482101a,35251,1732184000477 2024-11-21T10:13:21,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T10:13:21,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T10:13:21,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T10:13:21,616 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T10:13:21,618 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2651e482101a,35251,1732184000477 2024-11-21T10:13:21,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:21,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T10:13:21,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T10:13:21,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-21T10:13:21,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:21,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:21,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:21,653 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-21T10:13:21,654 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2651e482101a,35251,1732184000477 from backup master directory 2024-11-21T10:13:21,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T10:13:21,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2651e482101a,35251,1732184000477 2024-11-21T10:13:21,661 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T10:13:21,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T10:13:21,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-21T10:13:21,662 WARN [master/2651e482101a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T10:13:21,663 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2651e482101a,35251,1732184000477 2024-11-21T10:13:21,665 INFO [master/2651e482101a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-21T10:13:21,666 INFO [master/2651e482101a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-21T10:13:21,720 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/hbase.id] with ID: 114bcabb-0504-4c22-8f5b-f31aa098ec4f 2024-11-21T10:13:21,720 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/.tmp/hbase.id 2024-11-21T10:13:21,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741826_1002 (size=42) 2024-11-21T10:13:21,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741826_1002 (size=42) 2024-11-21T10:13:21,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741826_1002 (size=42) 2024-11-21T10:13:21,735 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/.tmp/hbase.id]:[hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/hbase.id] 2024-11-21T10:13:21,785 INFO [master/2651e482101a:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-21T10:13:21,791 INFO [master/2651e482101a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-21T10:13:21,809 INFO [master/2651e482101a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-21T10:13:21,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:21,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:21,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:21,819 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:21,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741827_1003 (size=196) 2024-11-21T10:13:21,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741827_1003 (size=196) 2024-11-21T10:13:21,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741827_1003 (size=196) 2024-11-21T10:13:21,855 INFO [master/2651e482101a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T10:13:21,856 INFO [master/2651e482101a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-21T10:13:21,870 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T10:13:21,873 INFO [master/2651e482101a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T10:13:21,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741828_1004 (size=1189) 2024-11-21T10:13:21,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741828_1004 (size=1189) 2024-11-21T10:13:21,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741828_1004 (size=1189) 2024-11-21T10:13:21,919 INFO [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store 2024-11-21T10:13:21,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741829_1005 (size=34) 2024-11-21T10:13:21,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741829_1005 (size=34) 2024-11-21T10:13:21,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741829_1005 (size=34) 2024-11-21T10:13:21,948 INFO [master/2651e482101a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-21T10:13:21,952 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:21,953 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T10:13:21,954 INFO [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T10:13:21,954 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T10:13:21,957 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T10:13:21,957 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T10:13:21,957 INFO [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T10:13:21,958 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732184001953Disabling compacts and flushes for region at 1732184001953Disabling writes for close at 1732184001957 (+4 ms)Writing region close event to WAL at 1732184001957Closed at 1732184001957 2024-11-21T10:13:21,961 WARN [master/2651e482101a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/.initializing 2024-11-21T10:13:21,961 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/WALs/2651e482101a,35251,1732184000477 2024-11-21T10:13:21,970 INFO [master/2651e482101a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T10:13:21,987 INFO [master/2651e482101a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2651e482101a%2C35251%2C1732184000477, suffix=, logDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/WALs/2651e482101a,35251,1732184000477, archiveDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/oldWALs, maxLogs=10 2024-11-21T10:13:22,014 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/WALs/2651e482101a,35251,1732184000477/2651e482101a%2C35251%2C1732184000477.1732184001991, exclude list is [], retry=0 2024-11-21T10:13:22,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39431,DS-3d33f7b2-5674-4258-8003-1641935bc33a,DISK] 2024-11-21T10:13:22,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-ebed3a63-7f40-4df1-bc25-cc54b1ace770,DISK] 2024-11-21T10:13:22,032 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37847,DS-5d8f2e7a-cf62-4175-abda-564b3f52e773,DISK] 2024-11-21T10:13:22,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-21T10:13:22,076 INFO [master/2651e482101a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/WALs/2651e482101a,35251,1732184000477/2651e482101a%2C35251%2C1732184000477.1732184001991 2024-11-21T10:13:22,077 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33461:33461),(127.0.0.1/127.0.0.1:44629:44629),(127.0.0.1/127.0.0.1:33871:33871)] 2024-11-21T10:13:22,078 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-21T10:13:22,078 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:22,083 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,084 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,120 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,145 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-21T10:13:22,149 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:22,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:22,153 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,156 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-21T10:13:22,157 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:22,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:22,159 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-21T10:13:22,162 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:22,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:22,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,167 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-21T10:13:22,167 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:22,168 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:22,169 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,173 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,174 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,180 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,181 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,184 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T10:13:22,188 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-21T10:13:22,194 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:22,195 INFO [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67532268, jitterRate=0.006309211254119873}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T10:13:22,203 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732184002096Initializing all the Stores at 1732184002098 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732184002099 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184002100 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184002100Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184002100Cleaning up temporary data from old regions at 1732184002181 (+81 ms)Region opened successfully at 1732184002203 (+22 ms) 2024-11-21T10:13:22,204 INFO [master/2651e482101a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-21T10:13:22,234 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5ca3c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2651e482101a/172.17.0.2:0 2024-11-21T10:13:22,262 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-21T10:13:22,272 INFO [master/2651e482101a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:22,273 INFO [master/2651e482101a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:22,275 INFO [master/2651e482101a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:22,276 INFO [master/2651e482101a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T10:13:22,281 INFO [master/2651e482101a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T10:13:22,281 INFO [master/2651e482101a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-21T10:13:22,306 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-21T10:13:22,314 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-21T10:13:22,356 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-21T10:13:22,359 INFO [master/2651e482101a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-21T10:13:22,361 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-21T10:13:22,373 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-21T10:13:22,377 INFO [master/2651e482101a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-21T10:13:22,383 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-21T10:13:22,394 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-21T10:13:22,396 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-21T10:13:22,409 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-21T10:13:22,430 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-21T10:13:22,440 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-21T10:13:22,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T10:13:22,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T10:13:22,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T10:13:22,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-21T10:13:22,452 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,455 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2651e482101a,35251,1732184000477, sessionid=0x1015ce240570000, setting cluster-up flag (Was=false) 2024-11-21T10:13:22,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,520 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-21T10:13:22,523 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2651e482101a,35251,1732184000477 2024-11-21T10:13:22,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,546 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:22,577 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-21T10:13:22,579 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2651e482101a,35251,1732184000477 2024-11-21T10:13:22,585 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-21T10:13:22,655 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:22,663 INFO [master/2651e482101a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-21T10:13:22,669 INFO [master/2651e482101a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-21T10:13:22,674 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2651e482101a,35251,1732184000477 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-21T10:13:22,681 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2651e482101a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T10:13:22,682 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2651e482101a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T10:13:22,682 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2651e482101a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T10:13:22,682 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2651e482101a:0, corePoolSize=5, maxPoolSize=5 2024-11-21T10:13:22,682 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2651e482101a:0, corePoolSize=10, maxPoolSize=10 2024-11-21T10:13:22,682 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,682 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2651e482101a:0, corePoolSize=2, maxPoolSize=2 2024-11-21T10:13:22,682 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,683 INFO [master/2651e482101a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184032683 2024-11-21T10:13:22,685 INFO [master/2651e482101a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-21T10:13:22,687 INFO [master/2651e482101a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-21T10:13:22,687 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(746): ClusterId : 114bcabb-0504-4c22-8f5b-f31aa098ec4f 2024-11-21T10:13:22,687 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(746): ClusterId : 114bcabb-0504-4c22-8f5b-f31aa098ec4f 2024-11-21T10:13:22,687 INFO [RS:2;2651e482101a:33515 {}] regionserver.HRegionServer(746): ClusterId : 114bcabb-0504-4c22-8f5b-f31aa098ec4f 2024-11-21T10:13:22,689 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:22,689 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-21T10:13:22,690 DEBUG [RS:2;2651e482101a:33515 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T10:13:22,690 DEBUG [RS:0;2651e482101a:36083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T10:13:22,690 DEBUG [RS:1;2651e482101a:36761 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-21T10:13:22,690 INFO [master/2651e482101a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-21T10:13:22,691 INFO [master/2651e482101a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-21T10:13:22,691 INFO [master/2651e482101a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-21T10:13:22,691 INFO [master/2651e482101a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-21T10:13:22,692 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,696 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:22,696 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-21T10:13:22,696 INFO [master/2651e482101a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-21T10:13:22,698 INFO [master/2651e482101a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-21T10:13:22,698 INFO [master/2651e482101a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-21T10:13:22,700 INFO [master/2651e482101a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-21T10:13:22,700 INFO [master/2651e482101a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-21T10:13:22,702 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2651e482101a:0:becomeActiveMaster-HFileCleaner.large.0-1732184002701,5,FailOnTimeoutGroup] 2024-11-21T10:13:22,703 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2651e482101a:0:becomeActiveMaster-HFileCleaner.small.0-1732184002702,5,FailOnTimeoutGroup] 2024-11-21T10:13:22,703 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,703 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-21T10:13:22,704 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,705 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741831_1007 (size=1321) 2024-11-21T10:13:22,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741831_1007 (size=1321) 2024-11-21T10:13:22,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741831_1007 (size=1321) 2024-11-21T10:13:22,713 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-21T10:13:22,714 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:22,716 DEBUG [RS:1;2651e482101a:36761 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T10:13:22,716 DEBUG [RS:0;2651e482101a:36083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T10:13:22,716 DEBUG [RS:2;2651e482101a:33515 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-21T10:13:22,716 DEBUG [RS:1;2651e482101a:36761 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T10:13:22,716 DEBUG [RS:2;2651e482101a:33515 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T10:13:22,716 DEBUG [RS:0;2651e482101a:36083 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-21T10:13:22,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741832_1008 (size=32) 2024-11-21T10:13:22,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741832_1008 (size=32) 2024-11-21T10:13:22,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741832_1008 (size=32) 2024-11-21T10:13:22,737 DEBUG [RS:0;2651e482101a:36083 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T10:13:22,737 DEBUG [RS:1;2651e482101a:36761 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T10:13:22,737 DEBUG [RS:0;2651e482101a:36083 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755b118d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2651e482101a/172.17.0.2:0 2024-11-21T10:13:22,737 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:22,737 DEBUG [RS:2;2651e482101a:33515 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-21T10:13:22,737 DEBUG [RS:1;2651e482101a:36761 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f47f6da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2651e482101a/172.17.0.2:0 2024-11-21T10:13:22,738 DEBUG [RS:2;2651e482101a:33515 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e4a872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2651e482101a/172.17.0.2:0 2024-11-21T10:13:22,746 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T10:13:22,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T10:13:22,750 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:22,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:22,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T10:13:22,753 DEBUG [RS:1;2651e482101a:36761 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;2651e482101a:36761 2024-11-21T10:13:22,753 DEBUG [RS:2;2651e482101a:33515 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;2651e482101a:33515 2024-11-21T10:13:22,755 DEBUG [RS:0;2651e482101a:36083 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2651e482101a:36083 2024-11-21T10:13:22,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T10:13:22,757 INFO [RS:0;2651e482101a:36083 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T10:13:22,757 INFO [RS:1;2651e482101a:36761 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T10:13:22,757 INFO [RS:2;2651e482101a:33515 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-21T10:13:22,757 INFO [RS:0;2651e482101a:36083 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T10:13:22,757 INFO [RS:1;2651e482101a:36761 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T10:13:22,757 INFO [RS:2;2651e482101a:33515 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-21T10:13:22,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:22,757 DEBUG [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T10:13:22,757 DEBUG [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T10:13:22,757 DEBUG [RS:2;2651e482101a:33515 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-21T10:13:22,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:22,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T10:13:22,761 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(2659): reportForDuty to master=2651e482101a,35251,1732184000477 with port=36083, startcode=1732184001229 2024-11-21T10:13:22,761 INFO [RS:2;2651e482101a:33515 {}] regionserver.HRegionServer(2659): reportForDuty to master=2651e482101a,35251,1732184000477 with port=33515, startcode=1732184001551 2024-11-21T10:13:22,761 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(2659): reportForDuty to master=2651e482101a,35251,1732184000477 with port=36761, startcode=1732184001491 2024-11-21T10:13:22,762 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T10:13:22,762 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:22,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:22,763 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T10:13:22,766 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T10:13:22,766 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:22,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:22,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T10:13:22,770 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740 2024-11-21T10:13:22,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740 2024-11-21T10:13:22,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T10:13:22,776 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T10:13:22,776 DEBUG [RS:1;2651e482101a:36761 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T10:13:22,776 DEBUG [RS:0;2651e482101a:36083 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T10:13:22,778 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T10:13:22,779 DEBUG [RS:2;2651e482101a:33515 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-21T10:13:22,782 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T10:13:22,794 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:22,800 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73613344, jitterRate=0.0969243049621582}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T10:13:22,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732184002738Initializing all the Stores at 1732184002740 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732184002740Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732184002746 (+6 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184002746Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732184002746Cleaning up temporary data from old regions at 1732184002776 (+30 ms)Region opened successfully at 1732184002805 (+29 ms) 2024-11-21T10:13:22,806 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T10:13:22,809 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T10:13:22,809 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T10:13:22,809 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T10:13:22,809 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T10:13:22,819 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T10:13:22,820 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732184002806Disabling compacts and flushes for region at 1732184002806Disabling writes for close at 1732184002809 (+3 ms)Writing region close event to WAL at 1732184002817 (+8 ms)Closed at 1732184002819 (+2 ms) 2024-11-21T10:13:22,822 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37473, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T10:13:22,822 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37803, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T10:13:22,825 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44187, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-21T10:13:22,827 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:22,827 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-21T10:13:22,830 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2651e482101a,36761,1732184001491 2024-11-21T10:13:22,833 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] master.ServerManager(517): Registering regionserver=2651e482101a,36761,1732184001491 2024-11-21T10:13:22,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T10:13:22,848 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2651e482101a,36083,1732184001229 2024-11-21T10:13:22,848 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] master.ServerManager(517): Registering regionserver=2651e482101a,36083,1732184001229 2024-11-21T10:13:22,853 DEBUG [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:22,853 DEBUG [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39887 2024-11-21T10:13:22,853 DEBUG [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T10:13:22,855 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:22,859 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2651e482101a,33515,1732184001551 2024-11-21T10:13:22,859 DEBUG [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:22,859 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] master.ServerManager(517): Registering regionserver=2651e482101a,33515,1732184001551 2024-11-21T10:13:22,859 DEBUG [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39887 2024-11-21T10:13:22,859 DEBUG [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T10:13:22,860 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-21T10:13:22,865 DEBUG [RS:2;2651e482101a:33515 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:22,865 DEBUG [RS:2;2651e482101a:33515 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:39887 2024-11-21T10:13:22,865 DEBUG [RS:2;2651e482101a:33515 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-21T10:13:22,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T10:13:22,907 DEBUG [RS:1;2651e482101a:36761 {}] zookeeper.ZKUtil(111): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2651e482101a,36761,1732184001491 2024-11-21T10:13:22,907 WARN [RS:1;2651e482101a:36761 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T10:13:22,907 DEBUG [RS:2;2651e482101a:33515 {}] zookeeper.ZKUtil(111): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2651e482101a,33515,1732184001551 2024-11-21T10:13:22,907 DEBUG [RS:0;2651e482101a:36083 {}] zookeeper.ZKUtil(111): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2651e482101a,36083,1732184001229 2024-11-21T10:13:22,908 WARN [RS:0;2651e482101a:36083 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T10:13:22,908 WARN [RS:2;2651e482101a:33515 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-21T10:13:22,908 INFO [RS:1;2651e482101a:36761 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T10:13:22,908 INFO [RS:0;2651e482101a:36083 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T10:13:22,908 INFO [RS:2;2651e482101a:33515 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T10:13:22,908 DEBUG [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,36761,1732184001491 2024-11-21T10:13:22,908 DEBUG [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,36083,1732184001229 2024-11-21T10:13:22,908 DEBUG [RS:2;2651e482101a:33515 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551 2024-11-21T10:13:22,909 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2651e482101a,36083,1732184001229] 2024-11-21T10:13:22,909 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2651e482101a,36761,1732184001491] 2024-11-21T10:13:22,909 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2651e482101a,33515,1732184001551] 2024-11-21T10:13:22,936 INFO [RS:0;2651e482101a:36083 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T10:13:22,936 INFO [RS:1;2651e482101a:36761 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T10:13:22,936 INFO [RS:2;2651e482101a:33515 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-21T10:13:22,952 INFO [RS:2;2651e482101a:33515 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T10:13:22,952 INFO [RS:0;2651e482101a:36083 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T10:13:22,954 INFO [RS:1;2651e482101a:36761 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-21T10:13:22,960 INFO [RS:0;2651e482101a:36083 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T10:13:22,960 INFO [RS:2;2651e482101a:33515 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T10:13:22,960 INFO [RS:1;2651e482101a:36761 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-21T10:13:22,960 INFO [RS:2;2651e482101a:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,960 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,960 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,961 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T10:13:22,965 INFO [RS:2;2651e482101a:33515 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T10:13:22,965 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-21T10:13:22,968 INFO [RS:0;2651e482101a:36083 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T10:13:22,968 INFO [RS:2;2651e482101a:33515 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T10:13:22,968 INFO [RS:1;2651e482101a:36761 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-21T10:13:22,970 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,970 INFO [RS:2;2651e482101a:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,970 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,971 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0, corePoolSize=2, maxPoolSize=2 2024-11-21T10:13:22,971 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,971 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0, corePoolSize=2, maxPoolSize=2 2024-11-21T10:13:22,972 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0, corePoolSize=2, maxPoolSize=2 2024-11-21T10:13:22,972 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2651e482101a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T10:13:22,972 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:2;2651e482101a:33515 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2651e482101a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T10:13:22,972 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,972 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2651e482101a:0, corePoolSize=1, maxPoolSize=1 2024-11-21T10:13:22,973 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2651e482101a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T10:13:22,973 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2651e482101a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T10:13:22,973 DEBUG [RS:0;2651e482101a:36083 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2651e482101a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T10:13:22,973 DEBUG [RS:1;2651e482101a:36761 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2651e482101a:0, corePoolSize=3, maxPoolSize=3 2024-11-21T10:13:22,978 INFO [RS:2;2651e482101a:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,978 INFO [RS:2;2651e482101a:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,979 INFO [RS:2;2651e482101a:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,979 INFO [RS:2;2651e482101a:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,979 INFO [RS:2;2651e482101a:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,979 INFO [RS:2;2651e482101a:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,33515,1732184001551-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T10:13:22,983 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,984 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,984 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,984 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,984 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,984 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,984 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,984 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,984 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,36761,1732184001491-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T10:13:22,984 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,984 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:22,985 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,36083,1732184001229-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T10:13:23,003 INFO [RS:2;2651e482101a:33515 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T10:13:23,003 INFO [RS:1;2651e482101a:36761 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T10:13:23,005 INFO [RS:2;2651e482101a:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,33515,1732184001551-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,005 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,36761,1732184001491-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,005 INFO [RS:2;2651e482101a:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,005 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,005 INFO [RS:2;2651e482101a:33515 {}] regionserver.Replication(171): 2651e482101a,33515,1732184001551 started 2024-11-21T10:13:23,005 INFO [RS:1;2651e482101a:36761 {}] regionserver.Replication(171): 2651e482101a,36761,1732184001491 started 2024-11-21T10:13:23,006 INFO [RS:0;2651e482101a:36083 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-21T10:13:23,006 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,36083,1732184001229-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,006 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,006 INFO [RS:0;2651e482101a:36083 {}] regionserver.Replication(171): 2651e482101a,36083,1732184001229 started 2024-11-21T10:13:23,011 WARN [2651e482101a:35251 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-21T10:13:23,024 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,024 INFO [RS:2;2651e482101a:33515 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,024 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,024 INFO [RS:2;2651e482101a:33515 {}] regionserver.HRegionServer(1482): Serving as 2651e482101a,33515,1732184001551, RpcServer on 2651e482101a/172.17.0.2:33515, sessionid=0x1015ce240570003 2024-11-21T10:13:23,024 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(1482): Serving as 2651e482101a,36761,1732184001491, RpcServer on 2651e482101a/172.17.0.2:36761, sessionid=0x1015ce240570002 2024-11-21T10:13:23,024 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(1482): Serving as 2651e482101a,36083,1732184001229, RpcServer on 2651e482101a/172.17.0.2:36083, sessionid=0x1015ce240570001 2024-11-21T10:13:23,025 DEBUG [RS:2;2651e482101a:33515 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T10:13:23,025 DEBUG [RS:1;2651e482101a:36761 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T10:13:23,025 DEBUG [RS:0;2651e482101a:36083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-21T10:13:23,025 DEBUG [RS:0;2651e482101a:36083 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2651e482101a,36083,1732184001229 2024-11-21T10:13:23,025 DEBUG [RS:1;2651e482101a:36761 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2651e482101a,36761,1732184001491 2024-11-21T10:13:23,025 DEBUG [RS:2;2651e482101a:33515 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2651e482101a,33515,1732184001551 2024-11-21T10:13:23,025 DEBUG [RS:0;2651e482101a:36083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2651e482101a,36083,1732184001229' 2024-11-21T10:13:23,025 DEBUG [RS:2;2651e482101a:33515 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2651e482101a,33515,1732184001551' 2024-11-21T10:13:23,025 DEBUG [RS:1;2651e482101a:36761 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2651e482101a,36761,1732184001491' 2024-11-21T10:13:23,025 DEBUG [RS:0;2651e482101a:36083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T10:13:23,025 DEBUG [RS:2;2651e482101a:33515 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T10:13:23,025 DEBUG [RS:1;2651e482101a:36761 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-21T10:13:23,026 DEBUG [RS:2;2651e482101a:33515 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T10:13:23,026 DEBUG [RS:0;2651e482101a:36083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T10:13:23,027 DEBUG [RS:0;2651e482101a:36083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T10:13:23,027 DEBUG [RS:2;2651e482101a:33515 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T10:13:23,027 DEBUG [RS:0;2651e482101a:36083 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T10:13:23,027 DEBUG [RS:2;2651e482101a:33515 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T10:13:23,027 DEBUG [RS:0;2651e482101a:36083 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2651e482101a,36083,1732184001229 2024-11-21T10:13:23,027 DEBUG [RS:2;2651e482101a:33515 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2651e482101a,33515,1732184001551 2024-11-21T10:13:23,027 DEBUG [RS:0;2651e482101a:36083 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2651e482101a,36083,1732184001229' 2024-11-21T10:13:23,027 DEBUG [RS:0;2651e482101a:36083 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T10:13:23,027 DEBUG [RS:2;2651e482101a:33515 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2651e482101a,33515,1732184001551' 2024-11-21T10:13:23,027 DEBUG [RS:2;2651e482101a:33515 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T10:13:23,028 DEBUG [RS:0;2651e482101a:36083 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T10:13:23,028 DEBUG [RS:2;2651e482101a:33515 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T10:13:23,028 DEBUG [RS:1;2651e482101a:36761 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-21T10:13:23,028 DEBUG [RS:2;2651e482101a:33515 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T10:13:23,028 INFO [RS:2;2651e482101a:33515 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T10:13:23,028 INFO [RS:2;2651e482101a:33515 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T10:13:23,029 DEBUG [RS:1;2651e482101a:36761 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-21T10:13:23,029 DEBUG [RS:1;2651e482101a:36761 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-21T10:13:23,029 DEBUG [RS:1;2651e482101a:36761 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2651e482101a,36761,1732184001491 2024-11-21T10:13:23,029 DEBUG [RS:1;2651e482101a:36761 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2651e482101a,36761,1732184001491' 2024-11-21T10:13:23,029 DEBUG [RS:1;2651e482101a:36761 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-21T10:13:23,030 DEBUG [RS:1;2651e482101a:36761 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-21T10:13:23,030 DEBUG [RS:1;2651e482101a:36761 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T10:13:23,030 INFO [RS:1;2651e482101a:36761 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T10:13:23,030 INFO [RS:1;2651e482101a:36761 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T10:13:23,032 DEBUG [RS:0;2651e482101a:36083 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-21T10:13:23,032 INFO [RS:0;2651e482101a:36083 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-21T10:13:23,032 INFO [RS:0;2651e482101a:36083 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-21T10:13:23,133 INFO [RS:2;2651e482101a:33515 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T10:13:23,133 INFO [RS:0;2651e482101a:36083 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T10:13:23,133 INFO [RS:1;2651e482101a:36761 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-21T10:13:23,136 INFO [RS:1;2651e482101a:36761 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2651e482101a%2C36761%2C1732184001491, suffix=, logDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,36761,1732184001491, archiveDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/oldWALs, maxLogs=32 2024-11-21T10:13:23,136 INFO [RS:0;2651e482101a:36083 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2651e482101a%2C36083%2C1732184001229, suffix=, logDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,36083,1732184001229, archiveDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/oldWALs, maxLogs=32 2024-11-21T10:13:23,136 INFO [RS:2;2651e482101a:33515 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2651e482101a%2C33515%2C1732184001551, suffix=, logDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551, archiveDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/oldWALs, maxLogs=32 2024-11-21T10:13:23,154 DEBUG [RS:1;2651e482101a:36761 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,36761,1732184001491/2651e482101a%2C36761%2C1732184001491.1732184003140, exclude list is [], retry=0 2024-11-21T10:13:23,157 DEBUG [RS:0;2651e482101a:36083 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,36083,1732184001229/2651e482101a%2C36083%2C1732184001229.1732184003140, exclude list is [], retry=0 2024-11-21T10:13:23,158 DEBUG [RS:2;2651e482101a:33515 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551/2651e482101a%2C33515%2C1732184001551.1732184003140, exclude list is [], retry=0 2024-11-21T10:13:23,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39431,DS-3d33f7b2-5674-4258-8003-1641935bc33a,DISK] 2024-11-21T10:13:23,160 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-ebed3a63-7f40-4df1-bc25-cc54b1ace770,DISK] 2024-11-21T10:13:23,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37847,DS-5d8f2e7a-cf62-4175-abda-564b3f52e773,DISK] 2024-11-21T10:13:23,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37847,DS-5d8f2e7a-cf62-4175-abda-564b3f52e773,DISK] 2024-11-21T10:13:23,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-ebed3a63-7f40-4df1-bc25-cc54b1ace770,DISK] 2024-11-21T10:13:23,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39431,DS-3d33f7b2-5674-4258-8003-1641935bc33a,DISK] 2024-11-21T10:13:23,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39431,DS-3d33f7b2-5674-4258-8003-1641935bc33a,DISK] 2024-11-21T10:13:23,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37847,DS-5d8f2e7a-cf62-4175-abda-564b3f52e773,DISK] 2024-11-21T10:13:23,190 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-ebed3a63-7f40-4df1-bc25-cc54b1ace770,DISK] 2024-11-21T10:13:23,195 INFO [RS:1;2651e482101a:36761 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,36761,1732184001491/2651e482101a%2C36761%2C1732184001491.1732184003140 2024-11-21T10:13:23,196 DEBUG [RS:1;2651e482101a:36761 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44629:44629),(127.0.0.1/127.0.0.1:33461:33461),(127.0.0.1/127.0.0.1:33871:33871)] 2024-11-21T10:13:23,197 INFO [RS:0;2651e482101a:36083 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,36083,1732184001229/2651e482101a%2C36083%2C1732184001229.1732184003140 2024-11-21T10:13:23,197 INFO [RS:2;2651e482101a:33515 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551/2651e482101a%2C33515%2C1732184001551.1732184003140 2024-11-21T10:13:23,197 DEBUG [RS:0;2651e482101a:36083 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33461:33461),(127.0.0.1/127.0.0.1:33871:33871),(127.0.0.1/127.0.0.1:44629:44629)] 2024-11-21T10:13:23,197 DEBUG [RS:2;2651e482101a:33515 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33871:33871),(127.0.0.1/127.0.0.1:44629:44629),(127.0.0.1/127.0.0.1:33461:33461)] 2024-11-21T10:13:23,265 DEBUG [2651e482101a:35251 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-21T10:13:23,274 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(204): Hosts are {2651e482101a=0} racks are {/default-rack=0} 2024-11-21T10:13:23,280 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T10:13:23,281 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T10:13:23,281 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-21T10:13:23,281 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T10:13:23,281 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T10:13:23,281 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T10:13:23,281 INFO [2651e482101a:35251 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T10:13:23,281 INFO [2651e482101a:35251 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T10:13:23,281 INFO [2651e482101a:35251 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T10:13:23,281 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T10:13:23,287 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:23,295 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,33515,1732184001551, state=OPENING 2024-11-21T10:13:23,346 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-21T10:13:23,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:23,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:23,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:23,357 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:23,358 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:23,358 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:23,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:23,359 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:23,361 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:23,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,33515,1732184001551}] 2024-11-21T10:13:23,540 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T10:13:23,542 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51337, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T10:13:23,553 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T10:13:23,554 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T10:13:23,554 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T10:13:23,558 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2651e482101a%2C33515%2C1732184001551.meta, suffix=.meta, logDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551, archiveDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/oldWALs, maxLogs=32 2024-11-21T10:13:23,573 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, exclude list is [], retry=0 2024-11-21T10:13:23,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39431,DS-3d33f7b2-5674-4258-8003-1641935bc33a,DISK] 2024-11-21T10:13:23,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37847,DS-5d8f2e7a-cf62-4175-abda-564b3f52e773,DISK] 2024-11-21T10:13:23,577 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-ebed3a63-7f40-4df1-bc25-cc54b1ace770,DISK] 2024-11-21T10:13:23,579 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta 2024-11-21T10:13:23,580 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44629:44629),(127.0.0.1/127.0.0.1:33871:33871),(127.0.0.1/127.0.0.1:33461:33461)] 2024-11-21T10:13:23,580 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T10:13:23,581 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T10:13:23,584 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T10:13:23,588 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T10:13:23,592 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T10:13:23,592 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:23,592 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T10:13:23,592 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T10:13:23,595 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T10:13:23,597 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T10:13:23,597 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:23,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:23,598 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T10:13:23,599 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T10:13:23,599 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:23,600 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:23,601 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T10:13:23,602 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T10:13:23,602 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:23,603 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:23,603 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T10:13:23,605 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T10:13:23,605 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:23,606 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:23,606 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T10:13:23,607 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740 2024-11-21T10:13:23,610 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740 2024-11-21T10:13:23,612 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T10:13:23,612 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T10:13:23,613 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T10:13:23,616 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T10:13:23,617 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75341247, jitterRate=0.12267206609249115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T10:13:23,618 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T10:13:23,619 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732184003593Writing region info on filesystem at 1732184003593Initializing all the Stores at 1732184003595 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732184003595Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732184003595Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184003595Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732184003595Cleaning up temporary data from old regions at 1732184003612 (+17 ms)Running coprocessor post-open hooks at 1732184003618 (+6 ms)Region opened successfully at 1732184003619 (+1 ms) 2024-11-21T10:13:23,625 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732184003532 2024-11-21T10:13:23,638 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T10:13:23,639 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T10:13:23,641 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:23,644 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,33515,1732184001551, state=OPEN 2024-11-21T10:13:23,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:23,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:23,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:23,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:23,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:23,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:23,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:23,658 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:23,658 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2651e482101a,33515,1732184001551 2024-11-21T10:13:23,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-21T10:13:23,664 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,33515,1732184001551 in 294 msec 2024-11-21T10:13:23,672 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-21T10:13:23,673 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 831 msec 2024-11-21T10:13:23,674 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:23,675 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-21T10:13:23,693 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T10:13:23,694 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1] 2024-11-21T10:13:23,713 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T10:13:23,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37487, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T10:13:23,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1220 sec 2024-11-21T10:13:23,736 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732184003736, completionTime=-1 2024-11-21T10:13:23,739 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-21T10:13:23,739 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:23,785 INFO [master/2651e482101a:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-21T10:13:23,785 INFO [master/2651e482101a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184063785 2024-11-21T10:13:23,786 INFO [master/2651e482101a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184123785 2024-11-21T10:13:23,786 INFO [master/2651e482101a:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 46 msec 2024-11-21T10:13:23,787 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:23,797 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,35251,1732184000477-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,797 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,35251,1732184000477-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,797 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,35251,1732184000477-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,799 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2651e482101a:35251, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,800 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,801 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,813 DEBUG [master/2651e482101a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-21T10:13:23,840 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.176sec 2024-11-21T10:13:23,842 INFO [master/2651e482101a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-21T10:13:23,843 INFO [master/2651e482101a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-21T10:13:23,844 INFO [master/2651e482101a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-21T10:13:23,844 INFO [master/2651e482101a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-21T10:13:23,844 INFO [master/2651e482101a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-21T10:13:23,845 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,35251,1732184000477-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-21T10:13:23,846 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,35251,1732184000477-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-21T10:13:23,850 DEBUG [master/2651e482101a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-21T10:13:23,851 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-21T10:13:23,852 INFO [master/2651e482101a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2651e482101a,35251,1732184000477-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-21T10:13:23,898 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9330f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T10:13:23,902 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-21T10:13:23,902 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-21T10:13:23,906 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2651e482101a,35251,-1 for getting cluster id 2024-11-21T10:13:23,908 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-21T10:13:23,917 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '114bcabb-0504-4c22-8f5b-f31aa098ec4f' 2024-11-21T10:13:23,920 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-21T10:13:23,920 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "114bcabb-0504-4c22-8f5b-f31aa098ec4f" 2024-11-21T10:13:23,920 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c217c0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T10:13:23,921 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2651e482101a,35251,-1] 2024-11-21T10:13:23,923 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-21T10:13:23,925 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T10:13:23,926 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47354, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-21T10:13:23,929 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6982d266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-21T10:13:23,929 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T10:13:23,936 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1] 2024-11-21T10:13:23,936 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T10:13:23,938 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T10:13:23,957 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2651e482101a,35251,1732184000477 2024-11-21T10:13:23,960 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-21T10:13:23,963 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2651e482101a,35251,1732184000477 2024-11-21T10:13:23,965 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@417d77fb 2024-11-21T10:13:23,966 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-21T10:13:23,968 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-21T10:13:23,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-21T10:13:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:23,987 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-21T10:13:23,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-21T10:13:23,989 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:23,992 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-21T10:13:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T10:13:24,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741837_1013 (size=358) 2024-11-21T10:13:24,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741837_1013 (size=358) 2024-11-21T10:13:24,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741837_1013 (size=358) 2024-11-21T10:13:24,010 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ed68f59ece209cbf633920e19d38f8f5, NAME => 'test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,010 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 60a0ac92d2da0863bf4f5f1aac0c8da3, NAME => 'test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,010 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 88fcaf42f6cd2d2a83b7023a1344f960, NAME => 'test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741839_1015 (size=42) 2024-11-21T10:13:24,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741839_1015 (size=42) 2024-11-21T10:13:24,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741839_1015 (size=42) 2024-11-21T10:13:24,029 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,030 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ed68f59ece209cbf633920e19d38f8f5, disabling compactions & flushes 2024-11-21T10:13:24,030 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:24,030 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:24,030 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. after waiting 0 ms 2024-11-21T10:13:24,030 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:24,030 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:24,030 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ed68f59ece209cbf633920e19d38f8f5: Waiting for close lock at 1732184004030Disabling compacts and flushes for region at 1732184004030Disabling writes for close at 1732184004030Writing region close event to WAL at 1732184004030Closed at 1732184004030 2024-11-21T10:13:24,031 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7b19ab03ffba010ed2e3bb6ecb72e84d, NAME => 'test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741838_1014 (size=45) 2024-11-21T10:13:24,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741838_1014 (size=45) 2024-11-21T10:13:24,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741838_1014 (size=45) 2024-11-21T10:13:24,037 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741840_1016 (size=45) 2024-11-21T10:13:24,037 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 88fcaf42f6cd2d2a83b7023a1344f960, disabling compactions & flushes 2024-11-21T10:13:24,037 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:24,038 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:24,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741840_1016 (size=45) 2024-11-21T10:13:24,038 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. after waiting 0 ms 2024-11-21T10:13:24,038 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:24,038 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:24,038 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 88fcaf42f6cd2d2a83b7023a1344f960: Waiting for close lock at 1732184004037Disabling compacts and flushes for region at 1732184004037Disabling writes for close at 1732184004038 (+1 ms)Writing region close event to WAL at 1732184004038Closed at 1732184004038 2024-11-21T10:13:24,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741840_1016 (size=45) 2024-11-21T10:13:24,039 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0fbe63f3c83c91920dd0c08974ab5ea8, NAME => 'test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,039 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,039 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 60a0ac92d2da0863bf4f5f1aac0c8da3, disabling compactions & flushes 2024-11-21T10:13:24,039 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3. 2024-11-21T10:13:24,039 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3. 2024-11-21T10:13:24,039 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3. after waiting 0 ms 2024-11-21T10:13:24,039 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3. 2024-11-21T10:13:24,039 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3. 2024-11-21T10:13:24,040 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 60a0ac92d2da0863bf4f5f1aac0c8da3: Waiting for close lock at 1732184004039Disabling compacts and flushes for region at 1732184004039Disabling writes for close at 1732184004039Writing region close event to WAL at 1732184004039Closed at 1732184004039 2024-11-21T10:13:24,040 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8b2fc4f9ae6f60de0b3c78d355788c60, NAME => 'test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741842_1018 (size=45) 2024-11-21T10:13:24,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741842_1018 (size=45) 2024-11-21T10:13:24,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741842_1018 (size=45) 2024-11-21T10:13:24,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741843_1019 (size=45) 2024-11-21T10:13:24,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741843_1019 (size=45) 2024-11-21T10:13:24,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741843_1019 (size=45) 2024-11-21T10:13:24,065 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,065 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,066 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0fbe63f3c83c91920dd0c08974ab5ea8, disabling compactions & flushes 2024-11-21T10:13:24,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8b2fc4f9ae6f60de0b3c78d355788c60, disabling compactions & flushes 2024-11-21T10:13:24,066 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:24,066 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:24,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:24,066 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:24,066 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. after waiting 0 ms 2024-11-21T10:13:24,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. after waiting 0 ms 2024-11-21T10:13:24,066 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:24,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:24,066 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:24,066 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:24,066 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8b2fc4f9ae6f60de0b3c78d355788c60: Waiting for close lock at 1732184004066Disabling compacts and flushes for region at 1732184004066Disabling writes for close at 1732184004066Writing region close event to WAL at 1732184004066Closed at 1732184004066 2024-11-21T10:13:24,066 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0fbe63f3c83c91920dd0c08974ab5ea8: Waiting for close lock at 1732184004066Disabling compacts and flushes for region at 1732184004066Disabling writes for close at 1732184004066Writing region close event to WAL at 1732184004066Closed at 1732184004066 2024-11-21T10:13:24,067 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 90d58495640c0c466573821e63530c34, NAME => 'test,hhh,1732184003970.90d58495640c0c466573821e63530c34.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,067 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 1cda3137e35dc414393695b95490205a, NAME => 'test,ggg,1732184003970.1cda3137e35dc414393695b95490205a.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741841_1017 (size=45) 2024-11-21T10:13:24,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741841_1017 (size=45) 2024-11-21T10:13:24,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741841_1017 (size=45) 2024-11-21T10:13:24,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 7b19ab03ffba010ed2e3bb6ecb72e84d, disabling compactions & flushes 2024-11-21T10:13:24,078 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:24,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:24,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. after waiting 0 ms 2024-11-21T10:13:24,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:24,078 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:24,078 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7b19ab03ffba010ed2e3bb6ecb72e84d: Waiting for close lock at 1732184004078Disabling compacts and flushes for region at 1732184004078Disabling writes for close at 1732184004078Writing region close event to WAL at 1732184004078Closed at 1732184004078 2024-11-21T10:13:24,079 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 04a9f41b735b6d4d1061d11055315003, NAME => 'test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741844_1020 (size=45) 2024-11-21T10:13:24,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741844_1020 (size=45) 2024-11-21T10:13:24,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741844_1020 (size=45) 2024-11-21T10:13:24,091 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,hhh,1732184003970.90d58495640c0c466573821e63530c34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,091 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 90d58495640c0c466573821e63530c34, disabling compactions & flushes 2024-11-21T10:13:24,091 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:24,091 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:24,091 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732184003970.90d58495640c0c466573821e63530c34. after waiting 0 ms 2024-11-21T10:13:24,091 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:24,091 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:24,091 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 90d58495640c0c466573821e63530c34: Waiting for close lock at 1732184004091Disabling compacts and flushes for region at 1732184004091Disabling writes for close at 1732184004091Writing region close event to WAL at 1732184004091Closed at 1732184004091 2024-11-21T10:13:24,092 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 34a8113c45096a8a44b690d6f96ba3b3, NAME => 'test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741845_1021 (size=45) 2024-11-21T10:13:24,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741845_1021 (size=45) 2024-11-21T10:13:24,099 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1732184003970.1cda3137e35dc414393695b95490205a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,099 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 1cda3137e35dc414393695b95490205a, disabling compactions & flushes 2024-11-21T10:13:24,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741845_1021 (size=45) 2024-11-21T10:13:24,099 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:24,099 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:24,099 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. after waiting 0 ms 2024-11-21T10:13:24,099 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:24,099 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:24,100 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 1cda3137e35dc414393695b95490205a: Waiting for close lock at 1732184004099Disabling compacts and flushes for region at 1732184004099Disabling writes for close at 1732184004099Writing region close event to WAL at 1732184004099Closed at 1732184004099 2024-11-21T10:13:24,100 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 52a49023e46e4cdb48a9866360492959, NAME => 'test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T10:13:24,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741846_1022 (size=45) 2024-11-21T10:13:24,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741846_1022 (size=45) 2024-11-21T10:13:24,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741846_1022 (size=45) 2024-11-21T10:13:24,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 04a9f41b735b6d4d1061d11055315003, disabling compactions & flushes 2024-11-21T10:13:24,109 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003. 2024-11-21T10:13:24,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003. 2024-11-21T10:13:24,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003. after waiting 0 ms 2024-11-21T10:13:24,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003. 2024-11-21T10:13:24,109 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003. 2024-11-21T10:13:24,109 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 04a9f41b735b6d4d1061d11055315003: Waiting for close lock at 1732184004109Disabling compacts and flushes for region at 1732184004109Disabling writes for close at 1732184004109Writing region close event to WAL at 1732184004109Closed at 1732184004109 2024-11-21T10:13:24,110 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f74cd0d96e89c26d17d931ba8b44a79d, NAME => 'test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741847_1023 (size=45) 2024-11-21T10:13:24,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741847_1023 (size=45) 2024-11-21T10:13:24,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741847_1023 (size=45) 2024-11-21T10:13:24,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,130 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 34a8113c45096a8a44b690d6f96ba3b3, disabling compactions & flushes 2024-11-21T10:13:24,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741848_1024 (size=45) 2024-11-21T10:13:24,131 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:24,131 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:24,131 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. after waiting 0 ms 2024-11-21T10:13:24,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741848_1024 (size=45) 2024-11-21T10:13:24,131 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:24,131 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:24,131 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 34a8113c45096a8a44b690d6f96ba3b3: Waiting for close lock at 1732184004130Disabling compacts and flushes for region at 1732184004130Disabling writes for close at 1732184004131 (+1 ms)Writing region close event to WAL at 1732184004131Closed at 1732184004131 2024-11-21T10:13:24,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741848_1024 (size=45) 2024-11-21T10:13:24,132 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b207ca8401450ab08698a420b743f234, NAME => 'test,mmm,1732184003970.b207ca8401450ab08698a420b743f234.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,133 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 52a49023e46e4cdb48a9866360492959, disabling compactions & flushes 2024-11-21T10:13:24,134 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:24,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:24,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. after waiting 0 ms 2024-11-21T10:13:24,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:24,134 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:24,134 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 52a49023e46e4cdb48a9866360492959: Waiting for close lock at 1732184004133Disabling compacts and flushes for region at 1732184004133Disabling writes for close at 1732184004134 (+1 ms)Writing region close event to WAL at 1732184004134Closed at 1732184004134 2024-11-21T10:13:24,135 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b5794ef6bcf8b6f343b167ad897a2aa0, NAME => 'test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741849_1025 (size=45) 2024-11-21T10:13:24,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741849_1025 (size=45) 2024-11-21T10:13:24,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741849_1025 (size=45) 2024-11-21T10:13:24,139 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,140 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f74cd0d96e89c26d17d931ba8b44a79d, disabling compactions & flushes 2024-11-21T10:13:24,140 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:24,140 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:24,140 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. after waiting 0 ms 2024-11-21T10:13:24,140 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:24,140 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:24,140 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f74cd0d96e89c26d17d931ba8b44a79d: Waiting for close lock at 1732184004140Disabling compacts and flushes for region at 1732184004140Disabling writes for close at 1732184004140Writing region close event to WAL at 1732184004140Closed at 1732184004140 2024-11-21T10:13:24,141 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ab23bbb87a53468249d295336f14f16f, NAME => 'test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741850_1026 (size=45) 2024-11-21T10:13:24,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741850_1026 (size=45) 2024-11-21T10:13:24,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741851_1027 (size=45) 2024-11-21T10:13:24,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741851_1027 (size=45) 2024-11-21T10:13:24,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741851_1027 (size=45) 2024-11-21T10:13:24,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741850_1026 (size=45) 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1732184003970.b207ca8401450ab08698a420b743f234.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b5794ef6bcf8b6f343b167ad897a2aa0, disabling compactions & flushes 2024-11-21T10:13:24,158 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b207ca8401450ab08698a420b743f234, disabling compactions & flushes 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:24,158 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. after waiting 0 ms 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. after waiting 0 ms 2024-11-21T10:13:24,158 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:24,158 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b5794ef6bcf8b6f343b167ad897a2aa0: Waiting for close lock at 1732184004158Disabling compacts and flushes for region at 1732184004158Disabling writes for close at 1732184004158Writing region close event to WAL at 1732184004158Closed at 1732184004158 2024-11-21T10:13:24,158 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b207ca8401450ab08698a420b743f234: Waiting for close lock at 1732184004158Disabling compacts and flushes for region at 1732184004158Disabling writes for close at 1732184004158Writing region close event to WAL at 1732184004158Closed at 1732184004158 2024-11-21T10:13:24,159 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => d0e3296740e89bbfb20e449acd71bdd4, NAME => 'test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,159 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => b1d292547267d9a1644f4ca59eacbdfa, NAME => 'test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741852_1028 (size=45) 2024-11-21T10:13:24,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741852_1028 (size=45) 2024-11-21T10:13:24,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741852_1028 (size=45) 2024-11-21T10:13:24,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ab23bbb87a53468249d295336f14f16f, disabling compactions & flushes 2024-11-21T10:13:24,176 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. 2024-11-21T10:13:24,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. 2024-11-21T10:13:24,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. after waiting 0 ms 2024-11-21T10:13:24,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. 2024-11-21T10:13:24,176 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. 2024-11-21T10:13:24,176 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ab23bbb87a53468249d295336f14f16f: Waiting for close lock at 1732184004176Disabling compacts and flushes for region at 1732184004176Disabling writes for close at 1732184004176Writing region close event to WAL at 1732184004176Closed at 1732184004176 2024-11-21T10:13:24,177 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 32979bec7fcef8d630309c08ee22a6e6, NAME => 'test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741854_1030 (size=45) 2024-11-21T10:13:24,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741854_1030 (size=45) 2024-11-21T10:13:24,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741854_1030 (size=45) 2024-11-21T10:13:24,189 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,189 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing b1d292547267d9a1644f4ca59eacbdfa, disabling compactions & flushes 2024-11-21T10:13:24,189 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:24,190 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:24,190 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. after waiting 0 ms 2024-11-21T10:13:24,190 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:24,190 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:24,190 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for b1d292547267d9a1644f4ca59eacbdfa: Waiting for close lock at 1732184004189Disabling compacts and flushes for region at 1732184004189Disabling writes for close at 1732184004190 (+1 ms)Writing region close event to WAL at 1732184004190Closed at 1732184004190 2024-11-21T10:13:24,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741853_1029 (size=45) 2024-11-21T10:13:24,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741853_1029 (size=45) 2024-11-21T10:13:24,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741853_1029 (size=45) 2024-11-21T10:13:24,191 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 9226917edc2a4eac226de198b54fce9c, NAME => 'test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,192 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,192 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing d0e3296740e89bbfb20e449acd71bdd4, disabling compactions & flushes 2024-11-21T10:13:24,192 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:24,192 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:24,193 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. after waiting 0 ms 2024-11-21T10:13:24,193 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:24,193 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:24,193 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for d0e3296740e89bbfb20e449acd71bdd4: Waiting for close lock at 1732184004192Disabling compacts and flushes for region at 1732184004192Disabling writes for close at 1732184004193 (+1 ms)Writing region close event to WAL at 1732184004193Closed at 1732184004193 2024-11-21T10:13:24,194 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => e90dd76cb8307eb049be31deaec171dd, NAME => 'test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741855_1031 (size=45) 2024-11-21T10:13:24,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741855_1031 (size=45) 2024-11-21T10:13:24,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741855_1031 (size=45) 2024-11-21T10:13:24,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 32979bec7fcef8d630309c08ee22a6e6, disabling compactions & flushes 2024-11-21T10:13:24,196 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:24,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:24,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. after waiting 0 ms 2024-11-21T10:13:24,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:24,196 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:24,196 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 32979bec7fcef8d630309c08ee22a6e6: Waiting for close lock at 1732184004196Disabling compacts and flushes for region at 1732184004196Disabling writes for close at 1732184004196Writing region close event to WAL at 1732184004196Closed at 1732184004196 2024-11-21T10:13:24,197 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => ba34ea0b7addcd538aef55c8823e35dc, NAME => 'test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741857_1033 (size=45) 2024-11-21T10:13:24,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741858_1034 (size=45) 2024-11-21T10:13:24,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741858_1034 (size=45) 2024-11-21T10:13:24,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing ba34ea0b7addcd538aef55c8823e35dc, disabling compactions & flushes 2024-11-21T10:13:24,212 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:24,212 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:24,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. after waiting 0 ms 2024-11-21T10:13:24,212 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing e90dd76cb8307eb049be31deaec171dd, disabling compactions & flushes 2024-11-21T10:13:24,212 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:24,212 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:24,212 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:24,212 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:24,212 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. after waiting 0 ms 2024-11-21T10:13:24,213 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for ba34ea0b7addcd538aef55c8823e35dc: Waiting for close lock at 1732184004212Disabling compacts and flushes for region at 1732184004212Disabling writes for close at 1732184004212Writing region close event to WAL at 1732184004212Closed at 1732184004212 2024-11-21T10:13:24,213 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:24,213 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:24,213 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for e90dd76cb8307eb049be31deaec171dd: Waiting for close lock at 1732184004212Disabling compacts and flushes for region at 1732184004212Disabling writes for close at 1732184004213 (+1 ms)Writing region close event to WAL at 1732184004213Closed at 1732184004213 2024-11-21T10:13:24,213 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fb8bd192054304ea3eed2270c166a773, NAME => 'test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741856_1032 (size=45) 2024-11-21T10:13:24,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741857_1033 (size=45) 2024-11-21T10:13:24,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741856_1032 (size=45) 2024-11-21T10:13:24,213 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 51fb43bd943dd75b745159a3a38956c8, NAME => 'test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741857_1033 (size=45) 2024-11-21T10:13:24,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741856_1032 (size=45) 2024-11-21T10:13:24,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741858_1034 (size=45) 2024-11-21T10:13:24,215 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,215 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 9226917edc2a4eac226de198b54fce9c, disabling compactions & flushes 2024-11-21T10:13:24,215 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c. 2024-11-21T10:13:24,215 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c. 2024-11-21T10:13:24,215 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c. after waiting 0 ms 2024-11-21T10:13:24,215 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c. 2024-11-21T10:13:24,215 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c. 2024-11-21T10:13:24,215 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 9226917edc2a4eac226de198b54fce9c: Waiting for close lock at 1732184004215Disabling compacts and flushes for region at 1732184004215Disabling writes for close at 1732184004215Writing region close event to WAL at 1732184004215Closed at 1732184004215 2024-11-21T10:13:24,216 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 12bdde3e5981435702484f9c042b20c9, NAME => 'test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741859_1035 (size=45) 2024-11-21T10:13:24,228 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741860_1036 (size=45) 2024-11-21T10:13:24,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741860_1036 (size=45) 2024-11-21T10:13:24,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741859_1035 (size=45) 2024-11-21T10:13:24,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741860_1036 (size=45) 2024-11-21T10:13:24,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741859_1035 (size=45) 2024-11-21T10:13:24,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741861_1037 (size=45) 2024-11-21T10:13:24,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741861_1037 (size=45) 2024-11-21T10:13:24,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741861_1037 (size=45) 2024-11-21T10:13:24,230 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,230 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,230 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 51fb43bd943dd75b745159a3a38956c8, disabling compactions & flushes 2024-11-21T10:13:24,230 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:24,230 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 12bdde3e5981435702484f9c042b20c9, disabling compactions & flushes 2024-11-21T10:13:24,231 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:24,231 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:24,231 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. after waiting 0 ms 2024-11-21T10:13:24,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:24,231 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:24,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. after waiting 0 ms 2024-11-21T10:13:24,231 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:24,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:24,231 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:24,231 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 51fb43bd943dd75b745159a3a38956c8: Waiting for close lock at 1732184004230Disabling compacts and flushes for region at 1732184004230Disabling writes for close at 1732184004231 (+1 ms)Writing region close event to WAL at 1732184004231Closed at 1732184004231 2024-11-21T10:13:24,231 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 12bdde3e5981435702484f9c042b20c9: Waiting for close lock at 1732184004230Disabling compacts and flushes for region at 1732184004230Disabling writes for close at 1732184004231 (+1 ms)Writing region close event to WAL at 1732184004231Closed at 1732184004231 2024-11-21T10:13:24,231 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 019a4c4ad79900668e68f2029ed4ec91, NAME => 'test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,232 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 460b55b3159e046531bc058dc9b7c499, NAME => 'test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff 2024-11-21T10:13:24,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741862_1038 (size=45) 2024-11-21T10:13:24,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741863_1039 (size=42) 2024-11-21T10:13:24,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741863_1039 (size=42) 2024-11-21T10:13:24,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741862_1038 (size=45) 2024-11-21T10:13:24,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741863_1039 (size=42) 2024-11-21T10:13:24,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741862_1038 (size=45) 2024-11-21T10:13:24,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T10:13:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T10:13:24,627 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,627 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing fb8bd192054304ea3eed2270c166a773, disabling compactions & flushes 2024-11-21T10:13:24,627 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:24,627 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:24,627 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. after waiting 0 ms 2024-11-21T10:13:24,627 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:24,627 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:24,627 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for fb8bd192054304ea3eed2270c166a773: Waiting for close lock at 1732184004627Disabling compacts and flushes for region at 1732184004627Disabling writes for close at 1732184004627Writing region close event to WAL at 1732184004627Closed at 1732184004627 2024-11-21T10:13:24,646 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:24,646 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 460b55b3159e046531bc058dc9b7c499, disabling compactions & flushes 2024-11-21T10:13:24,646 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:24,646 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:24,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 019a4c4ad79900668e68f2029ed4ec91, disabling compactions & flushes 2024-11-21T10:13:24,646 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. after waiting 0 ms 2024-11-21T10:13:24,646 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:24,646 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:24,646 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:24,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:24,646 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 460b55b3159e046531bc058dc9b7c499: Waiting for close lock at 1732184004646Disabling compacts and flushes for region at 1732184004646Disabling writes for close at 1732184004646Writing region close event to WAL at 1732184004646Closed at 1732184004646 2024-11-21T10:13:24,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. after waiting 0 ms 2024-11-21T10:13:24,646 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:24,647 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:24,647 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 019a4c4ad79900668e68f2029ed4ec91: Waiting for close lock at 1732184004646Disabling compacts and flushes for region at 1732184004646Disabling writes for close at 1732184004646Writing region close event to WAL at 1732184004647 (+1 ms)Closed at 1732184004647 2024-11-21T10:13:24,650 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-21T10:13:24,655 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,656 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732184003970.90d58495640c0c466573821e63530c34.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732184003970.1cda3137e35dc414393695b95490205a.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,657 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732184003970.b207ca8401450ab08698a420b743f234.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,658 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,659 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732184004650"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732184004650"}]},"ts":"1732184004650"} 2024-11-21T10:13:24,701 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-21T10:13:24,704 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-21T10:13:24,706 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732184004704"}]},"ts":"1732184004704"} 2024-11-21T10:13:24,711 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-21T10:13:24,712 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {2651e482101a=0} racks are {/default-rack=0} 2024-11-21T10:13:24,729 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T10:13:24,729 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T10:13:24,729 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-21T10:13:24,729 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T10:13:24,729 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T10:13:24,729 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T10:13:24,729 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T10:13:24,729 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T10:13:24,729 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T10:13:24,729 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T10:13:24,732 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ed68f59ece209cbf633920e19d38f8f5, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88fcaf42f6cd2d2a83b7023a1344f960, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0fbe63f3c83c91920dd0c08974ab5ea8, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8b2fc4f9ae6f60de0b3c78d355788c60, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1cda3137e35dc414393695b95490205a, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=90d58495640c0c466573821e63530c34, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=34a8113c45096a8a44b690d6f96ba3b3, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=52a49023e46e4cdb48a9866360492959, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b207ca8401450ab08698a420b743f234, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b5794ef6bcf8b6f343b167ad897a2aa0, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d0e3296740e89bbfb20e449acd71bdd4, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b1d292547267d9a1644f4ca59eacbdfa, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e90dd76cb8307eb049be31deaec171dd, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba34ea0b7addcd538aef55c8823e35dc, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51fb43bd943dd75b745159a3a38956c8, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12bdde3e5981435702484f9c042b20c9, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=019a4c4ad79900668e68f2029ed4ec91, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=460b55b3159e046531bc058dc9b7c499, ASSIGN}] 2024-11-21T10:13:24,741 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=460b55b3159e046531bc058dc9b7c499, ASSIGN 2024-11-21T10:13:24,742 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=460b55b3159e046531bc058dc9b7c499, ASSIGN; state=OFFLINE, location=2651e482101a,36083,1732184001229; forceNewPlan=false, retain=false 2024-11-21T10:13:24,743 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=019a4c4ad79900668e68f2029ed4ec91, ASSIGN 2024-11-21T10:13:24,745 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=019a4c4ad79900668e68f2029ed4ec91, ASSIGN; state=OFFLINE, location=2651e482101a,36761,1732184001491; forceNewPlan=false, retain=false 2024-11-21T10:13:24,746 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=12bdde3e5981435702484f9c042b20c9, ASSIGN 2024-11-21T10:13:24,747 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=12bdde3e5981435702484f9c042b20c9, ASSIGN; state=OFFLINE, location=2651e482101a,36761,1732184001491; forceNewPlan=false, retain=false 2024-11-21T10:13:24,748 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=51fb43bd943dd75b745159a3a38956c8, ASSIGN 2024-11-21T10:13:24,749 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=51fb43bd943dd75b745159a3a38956c8, ASSIGN; state=OFFLINE, location=2651e482101a,36083,1732184001229; forceNewPlan=false, retain=false 2024-11-21T10:13:24,751 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:24,752 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN; state=OFFLINE, location=2651e482101a,33515,1732184001551; forceNewPlan=false, retain=false 2024-11-21T10:13:24,753 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ba34ea0b7addcd538aef55c8823e35dc, ASSIGN 2024-11-21T10:13:24,754 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ba34ea0b7addcd538aef55c8823e35dc, ASSIGN; state=OFFLINE, location=2651e482101a,36083,1732184001229; forceNewPlan=false, retain=false 2024-11-21T10:13:24,755 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e90dd76cb8307eb049be31deaec171dd, ASSIGN 2024-11-21T10:13:24,756 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e90dd76cb8307eb049be31deaec171dd, ASSIGN; state=OFFLINE, location=2651e482101a,36761,1732184001491; forceNewPlan=false, retain=false 2024-11-21T10:13:24,758 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:24,759 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN; state=OFFLINE, location=2651e482101a,33515,1732184001551; forceNewPlan=false, retain=false 2024-11-21T10:13:24,760 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:24,761 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN; state=OFFLINE, location=2651e482101a,33515,1732184001551; forceNewPlan=false, retain=false 2024-11-21T10:13:24,763 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b1d292547267d9a1644f4ca59eacbdfa, ASSIGN 2024-11-21T10:13:24,764 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b1d292547267d9a1644f4ca59eacbdfa, ASSIGN; state=OFFLINE, location=2651e482101a,36083,1732184001229; forceNewPlan=false, retain=false 2024-11-21T10:13:24,765 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=d0e3296740e89bbfb20e449acd71bdd4, ASSIGN 2024-11-21T10:13:24,766 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=d0e3296740e89bbfb20e449acd71bdd4, ASSIGN; state=OFFLINE, location=2651e482101a,36761,1732184001491; forceNewPlan=false, retain=false 2024-11-21T10:13:24,768 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:24,769 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN; state=OFFLINE, location=2651e482101a,33515,1732184001551; forceNewPlan=false, retain=false 2024-11-21T10:13:24,770 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b5794ef6bcf8b6f343b167ad897a2aa0, ASSIGN 2024-11-21T10:13:24,771 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b5794ef6bcf8b6f343b167ad897a2aa0, ASSIGN; state=OFFLINE, location=2651e482101a,36083,1732184001229; forceNewPlan=false, retain=false 2024-11-21T10:13:24,773 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b207ca8401450ab08698a420b743f234, ASSIGN 2024-11-21T10:13:24,774 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b207ca8401450ab08698a420b743f234, ASSIGN; state=OFFLINE, location=2651e482101a,36761,1732184001491; forceNewPlan=false, retain=false 2024-11-21T10:13:24,776 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:24,777 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN; state=OFFLINE, location=2651e482101a,33515,1732184001551; forceNewPlan=false, retain=false 2024-11-21T10:13:24,779 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=52a49023e46e4cdb48a9866360492959, ASSIGN 2024-11-21T10:13:24,780 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=52a49023e46e4cdb48a9866360492959, ASSIGN; state=OFFLINE, location=2651e482101a,36761,1732184001491; forceNewPlan=false, retain=false 2024-11-21T10:13:24,781 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=34a8113c45096a8a44b690d6f96ba3b3, ASSIGN 2024-11-21T10:13:24,783 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=34a8113c45096a8a44b690d6f96ba3b3, ASSIGN; state=OFFLINE, location=2651e482101a,36083,1732184001229; forceNewPlan=false, retain=false 2024-11-21T10:13:24,784 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:24,785 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN; state=OFFLINE, location=2651e482101a,33515,1732184001551; forceNewPlan=false, retain=false 2024-11-21T10:13:24,786 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=90d58495640c0c466573821e63530c34, ASSIGN 2024-11-21T10:13:24,788 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=90d58495640c0c466573821e63530c34, ASSIGN; state=OFFLINE, location=2651e482101a,36083,1732184001229; forceNewPlan=false, retain=false 2024-11-21T10:13:24,789 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=1cda3137e35dc414393695b95490205a, ASSIGN 2024-11-21T10:13:24,790 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=1cda3137e35dc414393695b95490205a, ASSIGN; state=OFFLINE, location=2651e482101a,36761,1732184001491; forceNewPlan=false, retain=false 2024-11-21T10:13:24,791 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8b2fc4f9ae6f60de0b3c78d355788c60, ASSIGN 2024-11-21T10:13:24,793 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8b2fc4f9ae6f60de0b3c78d355788c60, ASSIGN; state=OFFLINE, location=2651e482101a,36761,1732184001491; forceNewPlan=false, retain=false 2024-11-21T10:13:24,794 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0fbe63f3c83c91920dd0c08974ab5ea8, ASSIGN 2024-11-21T10:13:24,795 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0fbe63f3c83c91920dd0c08974ab5ea8, ASSIGN; state=OFFLINE, location=2651e482101a,36083,1732184001229; forceNewPlan=false, retain=false 2024-11-21T10:13:24,796 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:24,797 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN; state=OFFLINE, location=2651e482101a,33515,1732184001551; forceNewPlan=false, retain=false 2024-11-21T10:13:24,798 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:24,799 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN; state=OFFLINE, location=2651e482101a,33515,1732184001551; forceNewPlan=false, retain=false 2024-11-21T10:13:24,801 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88fcaf42f6cd2d2a83b7023a1344f960, ASSIGN 2024-11-21T10:13:24,802 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88fcaf42f6cd2d2a83b7023a1344f960, ASSIGN; state=OFFLINE, location=2651e482101a,36761,1732184001491; forceNewPlan=false, retain=false 2024-11-21T10:13:24,803 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ed68f59ece209cbf633920e19d38f8f5, ASSIGN 2024-11-21T10:13:24,804 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ed68f59ece209cbf633920e19d38f8f5, ASSIGN; state=OFFLINE, location=2651e482101a,36083,1732184001229; forceNewPlan=false, retain=false 2024-11-21T10:13:24,896 INFO [2651e482101a:35251 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-21T10:13:24,897 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=019a4c4ad79900668e68f2029ed4ec91, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:24,898 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=12bdde3e5981435702484f9c042b20c9, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:24,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=e90dd76cb8307eb049be31deaec171dd, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:24,900 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:24,901 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=b207ca8401450ab08698a420b743f234, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:24,901 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=52a49023e46e4cdb48a9866360492959, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:24,902 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1cda3137e35dc414393695b95490205a, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:24,903 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=019a4c4ad79900668e68f2029ed4ec91, ASSIGN because future has completed 2024-11-21T10:13:24,903 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:24,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:24,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 019a4c4ad79900668e68f2029ed4ec91, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:24,907 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=12bdde3e5981435702484f9c042b20c9, ASSIGN because future has completed 2024-11-21T10:13:24,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e90dd76cb8307eb049be31deaec171dd, ASSIGN because future has completed 2024-11-21T10:13:24,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=d0e3296740e89bbfb20e449acd71bdd4, ASSIGN because future has completed 2024-11-21T10:13:24,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0e3296740e89bbfb20e449acd71bdd4, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:24,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b207ca8401450ab08698a420b743f234, ASSIGN because future has completed 2024-11-21T10:13:24,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=52a49023e46e4cdb48a9866360492959, ASSIGN because future has completed 2024-11-21T10:13:24,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=1cda3137e35dc414393695b95490205a, ASSIGN because future has completed 2024-11-21T10:13:24,918 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8b2fc4f9ae6f60de0b3c78d355788c60, ASSIGN because future has completed 2024-11-21T10:13:24,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b2fc4f9ae6f60de0b3c78d355788c60, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:24,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88fcaf42f6cd2d2a83b7023a1344f960, ASSIGN because future has completed 2024-11-21T10:13:24,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88fcaf42f6cd2d2a83b7023a1344f960, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:24,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cda3137e35dc414393695b95490205a, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:24,935 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52a49023e46e4cdb48a9866360492959, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:24,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure b207ca8401450ab08698a420b743f234, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:24,946 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure e90dd76cb8307eb049be31deaec171dd, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:24,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12bdde3e5981435702484f9c042b20c9, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:24,957 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=460b55b3159e046531bc058dc9b7c499, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:24,957 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=51fb43bd943dd75b745159a3a38956c8, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:24,958 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:24,959 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:24,960 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:24,960 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=460b55b3159e046531bc058dc9b7c499, ASSIGN because future has completed 2024-11-21T10:13:24,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 460b55b3159e046531bc058dc9b7c499, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:24,962 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=51fb43bd943dd75b745159a3a38956c8, ASSIGN because future has completed 2024-11-21T10:13:24,963 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ba34ea0b7addcd538aef55c8823e35dc, ASSIGN because future has completed 2024-11-21T10:13:24,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b1d292547267d9a1644f4ca59eacbdfa, ASSIGN because future has completed 2024-11-21T10:13:24,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure b1d292547267d9a1644f4ca59eacbdfa, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:24,967 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b5794ef6bcf8b6f343b167ad897a2aa0, ASSIGN because future has completed 2024-11-21T10:13:24,971 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5794ef6bcf8b6f343b167ad897a2aa0, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:24,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba34ea0b7addcd538aef55c8823e35dc, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:24,982 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51fb43bd943dd75b745159a3a38956c8, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:24,989 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:24,989 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=90d58495640c0c466573821e63530c34, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:24,990 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:24,991 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ed68f59ece209cbf633920e19d38f8f5, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:24,992 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:24,992 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:24,993 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:24,993 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:24,994 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=34a8113c45096a8a44b690d6f96ba3b3, ASSIGN because future has completed 2024-11-21T10:13:24,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34a8113c45096a8a44b690d6f96ba3b3, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:24,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=90d58495640c0c466573821e63530c34, ASSIGN because future has completed 2024-11-21T10:13:24,998 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0fbe63f3c83c91920dd0c08974ab5ea8, ASSIGN because future has completed 2024-11-21T10:13:24,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ed68f59ece209cbf633920e19d38f8f5, ASSIGN because future has completed 2024-11-21T10:13:25,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN because future has completed 2024-11-21T10:13:25,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed68f59ece209cbf633920e19d38f8f5, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:25,004 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN because future has completed 2024-11-21T10:13:25,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN because future has completed 2024-11-21T10:13:25,008 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN because future has completed 2024-11-21T10:13:25,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab23bbb87a53468249d295336f14f16f, server=2651e482101a,33515,1732184001551}] 2024-11-21T10:13:25,014 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,33515,1732184001551}] 2024-11-21T10:13:25,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,33515,1732184001551}] 2024-11-21T10:13:25,024 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,33515,1732184001551}] 2024-11-21T10:13:25,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fbe63f3c83c91920dd0c08974ab5ea8, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:25,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90d58495640c0c466573821e63530c34, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:25,036 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,037 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,038 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,039 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN because future has completed 2024-11-21T10:13:25,040 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,33515,1732184001551}] 2024-11-21T10:13:25,041 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN because future has completed 2024-11-21T10:13:25,043 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN because future has completed 2024-11-21T10:13:25,044 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN because future has completed 2024-11-21T10:13:25,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a0ac92d2da0863bf4f5f1aac0c8da3, server=2651e482101a,33515,1732184001551}] 2024-11-21T10:13:25,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,33515,1732184001551}] 2024-11-21T10:13:25,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,33515,1732184001551}] 2024-11-21T10:13:25,059 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T10:13:25,061 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45773, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T10:13:25,066 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:25,067 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => e90dd76cb8307eb049be31deaec171dd, NAME => 'test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-21T10:13:25,067 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:25,067 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,067 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:25,067 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:25,070 INFO [StoreOpener-e90dd76cb8307eb049be31deaec171dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:25,072 INFO [StoreOpener-e90dd76cb8307eb049be31deaec171dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e90dd76cb8307eb049be31deaec171dd columnFamilyName family 2024-11-21T10:13:25,072 DEBUG [StoreOpener-e90dd76cb8307eb049be31deaec171dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,073 INFO [StoreOpener-e90dd76cb8307eb049be31deaec171dd-1 {}] regionserver.HStore(327): Store=e90dd76cb8307eb049be31deaec171dd/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,073 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:25,075 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:25,075 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:25,076 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:25,076 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:25,079 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:25,081 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/e90dd76cb8307eb049be31deaec171dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,082 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened e90dd76cb8307eb049be31deaec171dd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66896675, jitterRate=-0.0031618624925613403}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,082 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:25,083 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for e90dd76cb8307eb049be31deaec171dd: Running coprocessor pre-open hook at 1732184005068Writing region info on filesystem at 1732184005068Initializing all the Stores at 1732184005069 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005069Cleaning up temporary data from old regions at 1732184005076 (+7 ms)Running coprocessor post-open hooks at 1732184005082 (+6 ms)Region opened successfully at 1732184005083 (+1 ms) 2024-11-21T10:13:25,085 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd., pid=38, masterSystemTime=1732184005059 2024-11-21T10:13:25,088 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:25,089 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:25,089 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:25,089 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => d0e3296740e89bbfb20e449acd71bdd4, NAME => 'test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-21T10:13:25,090 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:25,090 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:25,090 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,090 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:25,090 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:25,093 INFO [StoreOpener-d0e3296740e89bbfb20e449acd71bdd4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:25,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure e90dd76cb8307eb049be31deaec171dd, server=2651e482101a,36761,1732184001491 because future has completed 2024-11-21T10:13:25,095 INFO [StoreOpener-d0e3296740e89bbfb20e449acd71bdd4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d0e3296740e89bbfb20e449acd71bdd4 columnFamilyName family 2024-11-21T10:13:25,096 DEBUG [StoreOpener-d0e3296740e89bbfb20e449acd71bdd4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,096 INFO [StoreOpener-d0e3296740e89bbfb20e449acd71bdd4-1 {}] regionserver.HStore(327): Store=d0e3296740e89bbfb20e449acd71bdd4/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,097 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:25,097 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=24 2024-11-21T10:13:25,098 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure e90dd76cb8307eb049be31deaec171dd, server=2651e482101a,36761,1732184001491 in 148 msec 2024-11-21T10:13:25,098 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:25,099 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:25,099 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:25,100 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:25,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e90dd76cb8307eb049be31deaec171dd, ASSIGN in 367 msec 2024-11-21T10:13:25,102 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:25,105 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/d0e3296740e89bbfb20e449acd71bdd4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,106 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened d0e3296740e89bbfb20e449acd71bdd4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72584396, jitterRate=0.08159178495407104}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,106 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:25,106 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for d0e3296740e89bbfb20e449acd71bdd4: Running coprocessor pre-open hook at 1732184005090Writing region info on filesystem at 1732184005090Initializing all the Stores at 1732184005092 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005092Cleaning up temporary data from old regions at 1732184005100 (+8 ms)Running coprocessor post-open hooks at 1732184005106 (+6 ms)Region opened successfully at 1732184005106 2024-11-21T10:13:25,108 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4., pid=32, masterSystemTime=1732184005059 2024-11-21T10:13:25,110 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:25,110 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:25,111 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:25,111 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 8b2fc4f9ae6f60de0b3c78d355788c60, NAME => 'test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-21T10:13:25,112 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:25,112 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:25,112 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,112 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:25,112 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:25,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure d0e3296740e89bbfb20e449acd71bdd4, server=2651e482101a,36761,1732184001491 because future has completed 2024-11-21T10:13:25,115 INFO [StoreOpener-8b2fc4f9ae6f60de0b3c78d355788c60-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:25,115 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-21T10:13:25,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35105, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-21T10:13:25,118 INFO [StoreOpener-8b2fc4f9ae6f60de0b3c78d355788c60-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8b2fc4f9ae6f60de0b3c78d355788c60 columnFamilyName family 2024-11-21T10:13:25,118 DEBUG [StoreOpener-8b2fc4f9ae6f60de0b3c78d355788c60-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,119 INFO [StoreOpener-8b2fc4f9ae6f60de0b3c78d355788c60-1 {}] regionserver.HStore(327): Store=8b2fc4f9ae6f60de0b3c78d355788c60/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,119 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:25,120 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:25,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=20 2024-11-21T10:13:25,120 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure d0e3296740e89bbfb20e449acd71bdd4, server=2651e482101a,36761,1732184001491 in 206 msec 2024-11-21T10:13:25,121 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:25,122 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:25,122 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:25,123 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:25,123 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => ba34ea0b7addcd538aef55c8823e35dc, NAME => 'test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-21T10:13:25,124 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:25,124 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,124 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:25,124 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:25,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=d0e3296740e89bbfb20e449acd71bdd4, ASSIGN in 390 msec 2024-11-21T10:13:25,125 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:25,126 INFO [StoreOpener-ba34ea0b7addcd538aef55c8823e35dc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:25,128 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/8b2fc4f9ae6f60de0b3c78d355788c60/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,129 INFO [StoreOpener-ba34ea0b7addcd538aef55c8823e35dc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ba34ea0b7addcd538aef55c8823e35dc columnFamilyName family 2024-11-21T10:13:25,129 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 8b2fc4f9ae6f60de0b3c78d355788c60; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61220582, jitterRate=-0.08774223923683167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,129 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:25,129 DEBUG [StoreOpener-ba34ea0b7addcd538aef55c8823e35dc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,129 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 8b2fc4f9ae6f60de0b3c78d355788c60: Running coprocessor pre-open hook at 1732184005113Writing region info on filesystem at 1732184005113Initializing all the Stores at 1732184005115 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005115Cleaning up temporary data from old regions at 1732184005122 (+7 ms)Running coprocessor post-open hooks at 1732184005129 (+7 ms)Region opened successfully at 1732184005129 2024-11-21T10:13:25,130 INFO [StoreOpener-ba34ea0b7addcd538aef55c8823e35dc-1 {}] regionserver.HStore(327): Store=ba34ea0b7addcd538aef55c8823e35dc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,130 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:25,131 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60., pid=33, masterSystemTime=1732184005059 2024-11-21T10:13:25,131 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:25,132 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:25,133 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:25,133 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:25,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T10:13:25,134 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:25,134 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:25,134 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:25,134 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => b207ca8401450ab08698a420b743f234, NAME => 'test,mmm,1732184003970.b207ca8401450ab08698a420b743f234.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-21T10:13:25,135 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b207ca8401450ab08698a420b743f234 2024-11-21T10:13:25,135 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:25,135 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,mmm,1732184003970.b207ca8401450ab08698a420b743f234.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,135 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for b207ca8401450ab08698a420b743f234 2024-11-21T10:13:25,135 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for b207ca8401450ab08698a420b743f234 2024-11-21T10:13:25,136 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:25,138 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8b2fc4f9ae6f60de0b3c78d355788c60, server=2651e482101a,36761,1732184001491 because future has completed 2024-11-21T10:13:25,140 INFO [StoreOpener-b207ca8401450ab08698a420b743f234-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b207ca8401450ab08698a420b743f234 2024-11-21T10:13:25,141 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ba34ea0b7addcd538aef55c8823e35dc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,141 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened ba34ea0b7addcd538aef55c8823e35dc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70990957, jitterRate=0.05784769356250763}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,142 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:25,142 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for ba34ea0b7addcd538aef55c8823e35dc: Running coprocessor pre-open hook at 1732184005124Writing region info on filesystem at 1732184005124Initializing all the Stores at 1732184005125 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005126 (+1 ms)Cleaning up temporary data from old regions at 1732184005133 (+7 ms)Running coprocessor post-open hooks at 1732184005142 (+9 ms)Region opened successfully at 1732184005142 2024-11-21T10:13:25,142 INFO [StoreOpener-b207ca8401450ab08698a420b743f234-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b207ca8401450ab08698a420b743f234 columnFamilyName family 2024-11-21T10:13:25,143 DEBUG [StoreOpener-b207ca8401450ab08698a420b743f234-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,143 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc., pid=43, masterSystemTime=1732184005115 2024-11-21T10:13:25,144 INFO [StoreOpener-b207ca8401450ab08698a420b743f234-1 {}] regionserver.HStore(327): Store=b207ca8401450ab08698a420b743f234/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,144 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for b207ca8401450ab08698a420b743f234 2024-11-21T10:13:25,145 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b207ca8401450ab08698a420b743f234 2024-11-21T10:13:25,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=10 2024-11-21T10:13:25,146 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 8b2fc4f9ae6f60de0b3c78d355788c60, server=2651e482101a,36761,1732184001491 in 221 msec 2024-11-21T10:13:25,146 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b207ca8401450ab08698a420b743f234 2024-11-21T10:13:25,147 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:25,147 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:25,147 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for b207ca8401450ab08698a420b743f234 2024-11-21T10:13:25,147 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for b207ca8401450ab08698a420b743f234 2024-11-21T10:13:25,147 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:25,147 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 0fbe63f3c83c91920dd0c08974ab5ea8, NAME => 'test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-21T10:13:25,148 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:25,148 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,148 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:25,148 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:25,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8b2fc4f9ae6f60de0b3c78d355788c60, ASSIGN in 415 msec 2024-11-21T10:13:25,150 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for b207ca8401450ab08698a420b743f234 2024-11-21T10:13:25,151 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:25,151 INFO [StoreOpener-0fbe63f3c83c91920dd0c08974ab5ea8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:25,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure ba34ea0b7addcd538aef55c8823e35dc, server=2651e482101a,36083,1732184001229 because future has completed 2024-11-21T10:13:25,155 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b207ca8401450ab08698a420b743f234/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,155 INFO [StoreOpener-0fbe63f3c83c91920dd0c08974ab5ea8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0fbe63f3c83c91920dd0c08974ab5ea8 columnFamilyName family 2024-11-21T10:13:25,155 DEBUG [StoreOpener-0fbe63f3c83c91920dd0c08974ab5ea8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,156 INFO [StoreOpener-0fbe63f3c83c91920dd0c08974ab5ea8-1 {}] regionserver.HStore(327): Store=0fbe63f3c83c91920dd0c08974ab5ea8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,156 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened b207ca8401450ab08698a420b743f234; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73764603, jitterRate=0.0991782397031784}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,156 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b207ca8401450ab08698a420b743f234 2024-11-21T10:13:25,156 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for b207ca8401450ab08698a420b743f234: Running coprocessor pre-open hook at 1732184005135Writing region info on filesystem at 1732184005135Initializing all the Stores at 1732184005137 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005137Cleaning up temporary data from old regions at 1732184005147 (+10 ms)Running coprocessor post-open hooks at 1732184005156 (+9 ms)Region opened successfully at 1732184005156 2024-11-21T10:13:25,156 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:25,158 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:25,158 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732184003970.b207ca8401450ab08698a420b743f234., pid=37, masterSystemTime=1732184005059 2024-11-21T10:13:25,159 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:25,160 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:25,160 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:25,162 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:25,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=25 2024-11-21T10:13:25,162 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:25,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure ba34ea0b7addcd538aef55c8823e35dc, server=2651e482101a,36083,1732184001229 in 180 msec 2024-11-21T10:13:25,162 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:25,162 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 019a4c4ad79900668e68f2029ed4ec91, NAME => 'test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-21T10:13:25,163 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:25,163 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,163 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:25,164 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:25,164 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:25,166 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ba34ea0b7addcd538aef55c8823e35dc, ASSIGN in 431 msec 2024-11-21T10:13:25,167 INFO [StoreOpener-019a4c4ad79900668e68f2029ed4ec91-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:25,170 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=b207ca8401450ab08698a420b743f234, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:25,170 INFO [StoreOpener-019a4c4ad79900668e68f2029ed4ec91-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 019a4c4ad79900668e68f2029ed4ec91 columnFamilyName family 2024-11-21T10:13:25,170 DEBUG [StoreOpener-019a4c4ad79900668e68f2029ed4ec91-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,171 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/0fbe63f3c83c91920dd0c08974ab5ea8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,172 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:25,172 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 32979bec7fcef8d630309c08ee22a6e6, NAME => 'test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-21T10:13:25,172 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:25,173 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,173 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:25,173 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 0fbe63f3c83c91920dd0c08974ab5ea8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65043334, jitterRate=-0.03077879548072815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,173 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:25,173 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:25,173 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 0fbe63f3c83c91920dd0c08974ab5ea8: Running coprocessor pre-open hook at 1732184005149Writing region info on filesystem at 1732184005149Initializing all the Stores at 1732184005150 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005150Cleaning up temporary data from old regions at 1732184005160 (+10 ms)Running coprocessor post-open hooks at 1732184005173 (+13 ms)Region opened successfully at 1732184005173 2024-11-21T10:13:25,174 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8., pid=51, masterSystemTime=1732184005115 2024-11-21T10:13:25,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure b207ca8401450ab08698a420b743f234, server=2651e482101a,36761,1732184001491 because future has completed 2024-11-21T10:13:25,174 INFO [StoreOpener-019a4c4ad79900668e68f2029ed4ec91-1 {}] regionserver.HStore(327): Store=019a4c4ad79900668e68f2029ed4ec91/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,175 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:25,176 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:25,177 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:25,178 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:25,178 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:25,179 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:25,179 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:25,179 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:25,179 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => ed68f59ece209cbf633920e19d38f8f5, NAME => 'test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-21T10:13:25,180 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:25,180 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,180 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:25,180 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:25,182 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:25,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=17 2024-11-21T10:13:25,184 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure b207ca8401450ab08698a420b743f234, server=2651e482101a,36761,1732184001491 in 237 msec 2024-11-21T10:13:25,185 INFO [StoreOpener-32979bec7fcef8d630309c08ee22a6e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:25,186 INFO [StoreOpener-ed68f59ece209cbf633920e19d38f8f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:25,188 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b207ca8401450ab08698a420b743f234, ASSIGN in 453 msec 2024-11-21T10:13:25,188 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/019a4c4ad79900668e68f2029ed4ec91/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,188 INFO [StoreOpener-32979bec7fcef8d630309c08ee22a6e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32979bec7fcef8d630309c08ee22a6e6 columnFamilyName family 2024-11-21T10:13:25,189 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 019a4c4ad79900668e68f2029ed4ec91; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59601803, jitterRate=-0.11186392605304718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,189 DEBUG [StoreOpener-32979bec7fcef8d630309c08ee22a6e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,189 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:25,190 INFO [StoreOpener-ed68f59ece209cbf633920e19d38f8f5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ed68f59ece209cbf633920e19d38f8f5 columnFamilyName family 2024-11-21T10:13:25,190 DEBUG [StoreOpener-ed68f59ece209cbf633920e19d38f8f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,190 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 019a4c4ad79900668e68f2029ed4ec91: Running coprocessor pre-open hook at 1732184005164Writing region info on filesystem at 1732184005164Initializing all the Stores at 1732184005166 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005166Cleaning up temporary data from old regions at 1732184005178 (+12 ms)Running coprocessor post-open hooks at 1732184005190 (+12 ms)Region opened successfully at 1732184005190 2024-11-21T10:13:25,190 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:25,191 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91., pid=31, masterSystemTime=1732184005059 2024-11-21T10:13:25,193 INFO [StoreOpener-ed68f59ece209cbf633920e19d38f8f5-1 {}] regionserver.HStore(327): Store=ed68f59ece209cbf633920e19d38f8f5/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,193 INFO [StoreOpener-32979bec7fcef8d630309c08ee22a6e6-1 {}] regionserver.HStore(327): Store=32979bec7fcef8d630309c08ee22a6e6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,193 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:25,193 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:25,195 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0fbe63f3c83c91920dd0c08974ab5ea8, server=2651e482101a,36083,1732184001229 because future has completed 2024-11-21T10:13:25,195 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:25,195 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:25,196 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:25,196 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:25,196 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:25,196 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:25,196 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 52a49023e46e4cdb48a9866360492959, NAME => 'test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-21T10:13:25,196 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:25,197 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:25,197 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,197 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:25,197 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:25,197 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:25,198 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:25,198 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:25,199 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:25,199 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:25,200 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 019a4c4ad79900668e68f2029ed4ec91, server=2651e482101a,36761,1732184001491 because future has completed 2024-11-21T10:13:25,201 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:25,202 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:25,204 INFO [StoreOpener-52a49023e46e4cdb48a9866360492959-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:25,207 INFO [StoreOpener-52a49023e46e4cdb48a9866360492959-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52a49023e46e4cdb48a9866360492959 columnFamilyName family 2024-11-21T10:13:25,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=9 2024-11-21T10:13:25,207 DEBUG [StoreOpener-52a49023e46e4cdb48a9866360492959-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,207 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 0fbe63f3c83c91920dd0c08974ab5ea8, server=2651e482101a,36083,1732184001229 in 174 msec 2024-11-21T10:13:25,208 INFO [StoreOpener-52a49023e46e4cdb48a9866360492959-1 {}] regionserver.HStore(327): Store=52a49023e46e4cdb48a9866360492959/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,208 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:25,209 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:25,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0fbe63f3c83c91920dd0c08974ab5ea8, ASSIGN in 476 msec 2024-11-21T10:13:25,210 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:25,211 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:25,211 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:25,215 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:25,216 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/32979bec7fcef8d630309c08ee22a6e6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,218 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ed68f59ece209cbf633920e19d38f8f5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-21T10:13:25,218 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 019a4c4ad79900668e68f2029ed4ec91, server=2651e482101a,36761,1732184001491 in 307 msec 2024-11-21T10:13:25,218 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened ed68f59ece209cbf633920e19d38f8f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60696357, jitterRate=-0.09555380046367645}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,218 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 32979bec7fcef8d630309c08ee22a6e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68401660, jitterRate=0.019264161586761475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,218 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:25,218 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:25,219 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for ed68f59ece209cbf633920e19d38f8f5: Running coprocessor pre-open hook at 1732184005180Writing region info on filesystem at 1732184005180Initializing all the Stores at 1732184005182 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005182Cleaning up temporary data from old regions at 1732184005198 (+16 ms)Running coprocessor post-open hooks at 1732184005218 (+20 ms)Region opened successfully at 1732184005218 2024-11-21T10:13:25,219 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 32979bec7fcef8d630309c08ee22a6e6: Running coprocessor pre-open hook at 1732184005173Writing region info on filesystem at 1732184005173Initializing all the Stores at 1732184005175 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005175Cleaning up temporary data from old regions at 1732184005199 (+24 ms)Running coprocessor post-open hooks at 1732184005218 (+19 ms)Region opened successfully at 1732184005218 2024-11-21T10:13:25,220 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6., pid=48, masterSystemTime=1732184005163 2024-11-21T10:13:25,220 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5., pid=46, masterSystemTime=1732184005115 2024-11-21T10:13:25,222 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/52a49023e46e4cdb48a9866360492959/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=019a4c4ad79900668e68f2029ed4ec91, ASSIGN in 487 msec 2024-11-21T10:13:25,224 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 52a49023e46e4cdb48a9866360492959; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63389269, jitterRate=-0.05542628467082977}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,224 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:25,224 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 52a49023e46e4cdb48a9866360492959: Running coprocessor pre-open hook at 1732184005197Writing region info on filesystem at 1732184005197Initializing all the Stores at 1732184005200 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005200Cleaning up temporary data from old regions at 1732184005211 (+11 ms)Running coprocessor post-open hooks at 1732184005224 (+13 ms)Region opened successfully at 1732184005224 2024-11-21T10:13:25,225 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:25,225 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:25,225 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:25,225 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=32979bec7fcef8d630309c08ee22a6e6, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,225 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => f74cd0d96e89c26d17d931ba8b44a79d, NAME => 'test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-21T10:13:25,226 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:25,226 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:25,226 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:25,226 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,226 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:25,226 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:25,226 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:25,226 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 90d58495640c0c466573821e63530c34, NAME => 'test,hhh,1732184003970.90d58495640c0c466573821e63530c34.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-21T10:13:25,227 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 90d58495640c0c466573821e63530c34 2024-11-21T10:13:25,227 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,hhh,1732184003970.90d58495640c0c466573821e63530c34.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,227 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 90d58495640c0c466573821e63530c34 2024-11-21T10:13:25,227 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 90d58495640c0c466573821e63530c34 2024-11-21T10:13:25,228 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959., pid=36, masterSystemTime=1732184005059 2024-11-21T10:13:25,229 INFO [StoreOpener-f74cd0d96e89c26d17d931ba8b44a79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:25,230 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,33515,1732184001551 because future has completed 2024-11-21T10:13:25,232 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:25,232 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:25,232 INFO [StoreOpener-f74cd0d96e89c26d17d931ba8b44a79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f74cd0d96e89c26d17d931ba8b44a79d columnFamilyName family 2024-11-21T10:13:25,232 DEBUG [StoreOpener-f74cd0d96e89c26d17d931ba8b44a79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,232 INFO [StoreOpener-90d58495640c0c466573821e63530c34-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 90d58495640c0c466573821e63530c34 2024-11-21T10:13:25,232 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:25,233 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 1cda3137e35dc414393695b95490205a, NAME => 'test,ggg,1732184003970.1cda3137e35dc414393695b95490205a.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-21T10:13:25,233 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 1cda3137e35dc414393695b95490205a 2024-11-21T10:13:25,233 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ggg,1732184003970.1cda3137e35dc414393695b95490205a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,233 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 1cda3137e35dc414393695b95490205a 2024-11-21T10:13:25,233 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 1cda3137e35dc414393695b95490205a 2024-11-21T10:13:25,233 INFO [StoreOpener-f74cd0d96e89c26d17d931ba8b44a79d-1 {}] regionserver.HStore(327): Store=f74cd0d96e89c26d17d931ba8b44a79d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,234 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:25,235 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:25,236 INFO [StoreOpener-90d58495640c0c466573821e63530c34-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 90d58495640c0c466573821e63530c34 columnFamilyName family 2024-11-21T10:13:25,236 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:25,236 DEBUG [StoreOpener-90d58495640c0c466573821e63530c34-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,236 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:25,236 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:25,236 INFO [StoreOpener-90d58495640c0c466573821e63530c34-1 {}] regionserver.HStore(327): Store=90d58495640c0c466573821e63530c34/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,237 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 90d58495640c0c466573821e63530c34 2024-11-21T10:13:25,237 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/90d58495640c0c466573821e63530c34 2024-11-21T10:13:25,238 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/90d58495640c0c466573821e63530c34 2024-11-21T10:13:25,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=22 2024-11-21T10:13:25,239 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 90d58495640c0c466573821e63530c34 2024-11-21T10:13:25,239 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 90d58495640c0c466573821e63530c34 2024-11-21T10:13:25,239 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,33515,1732184001551 in 218 msec 2024-11-21T10:13:25,239 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:25,241 INFO [StoreOpener-1cda3137e35dc414393695b95490205a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 1cda3137e35dc414393695b95490205a 2024-11-21T10:13:25,242 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 90d58495640c0c466573821e63530c34 2024-11-21T10:13:25,243 INFO [StoreOpener-1cda3137e35dc414393695b95490205a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1cda3137e35dc414393695b95490205a columnFamilyName family 2024-11-21T10:13:25,243 DEBUG [StoreOpener-1cda3137e35dc414393695b95490205a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,243 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN in 508 msec 2024-11-21T10:13:25,244 INFO [StoreOpener-1cda3137e35dc414393695b95490205a-1 {}] regionserver.HStore(327): Store=1cda3137e35dc414393695b95490205a/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,244 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 1cda3137e35dc414393695b95490205a 2024-11-21T10:13:25,245 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=52a49023e46e4cdb48a9866360492959, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:25,248 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/1cda3137e35dc414393695b95490205a 2024-11-21T10:13:25,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52a49023e46e4cdb48a9866360492959, server=2651e482101a,36761,1732184001491 because future has completed 2024-11-21T10:13:25,249 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/1cda3137e35dc414393695b95490205a 2024-11-21T10:13:25,249 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:25,250 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 1cda3137e35dc414393695b95490205a 2024-11-21T10:13:25,250 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 1cda3137e35dc414393695b95490205a 2024-11-21T10:13:25,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure ed68f59ece209cbf633920e19d38f8f5, server=2651e482101a,36083,1732184001229 because future has completed 2024-11-21T10:13:25,254 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 1cda3137e35dc414393695b95490205a 2024-11-21T10:13:25,258 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/90d58495640c0c466573821e63530c34/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,258 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/1cda3137e35dc414393695b95490205a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,259 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 90d58495640c0c466573821e63530c34; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62502955, jitterRate=-0.06863339245319366}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,259 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 90d58495640c0c466573821e63530c34 2024-11-21T10:13:25,259 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 90d58495640c0c466573821e63530c34: Running coprocessor pre-open hook at 1732184005227Writing region info on filesystem at 1732184005228 (+1 ms)Initializing all the Stores at 1732184005230 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005230Cleaning up temporary data from old regions at 1732184005239 (+9 ms)Running coprocessor post-open hooks at 1732184005259 (+20 ms)Region opened successfully at 1732184005259 2024-11-21T10:13:25,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=15 2024-11-21T10:13:25,260 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure 52a49023e46e4cdb48a9866360492959, server=2651e482101a,36761,1732184001491 in 319 msec 2024-11-21T10:13:25,260 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 1cda3137e35dc414393695b95490205a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69312371, jitterRate=0.03283481299877167}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,260 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1cda3137e35dc414393695b95490205a 2024-11-21T10:13:25,260 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 1cda3137e35dc414393695b95490205a: Running coprocessor pre-open hook at 1732184005233Writing region info on filesystem at 1732184005233Initializing all the Stores at 1732184005235 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005235Cleaning up temporary data from old regions at 1732184005250 (+15 ms)Running coprocessor post-open hooks at 1732184005260 (+10 ms)Region opened successfully at 1732184005260 2024-11-21T10:13:25,260 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732184003970.90d58495640c0c466573821e63530c34., pid=52, masterSystemTime=1732184005115 2024-11-21T10:13:25,261 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732184003970.1cda3137e35dc414393695b95490205a., pid=35, masterSystemTime=1732184005059 2024-11-21T10:13:25,263 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:25,264 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:25,264 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:25,264 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=52a49023e46e4cdb48a9866360492959, ASSIGN in 529 msec 2024-11-21T10:13:25,264 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => b1d292547267d9a1644f4ca59eacbdfa, NAME => 'test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-21T10:13:25,265 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:25,265 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,265 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:25,265 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:25,266 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=90d58495640c0c466573821e63530c34, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:25,267 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:25,267 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:25,267 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:25,267 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 12bdde3e5981435702484f9c042b20c9, NAME => 'test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-21T10:13:25,268 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:25,268 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,268 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/f74cd0d96e89c26d17d931ba8b44a79d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,268 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:25,268 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:25,269 INFO [StoreOpener-b1d292547267d9a1644f4ca59eacbdfa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:25,270 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened f74cd0d96e89c26d17d931ba8b44a79d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71154637, jitterRate=0.060286715626716614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,270 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:25,270 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for f74cd0d96e89c26d17d931ba8b44a79d: Running coprocessor pre-open hook at 1732184005226Writing region info on filesystem at 1732184005226Initializing all the Stores at 1732184005229 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005229Cleaning up temporary data from old regions at 1732184005236 (+7 ms)Running coprocessor post-open hooks at 1732184005270 (+34 ms)Region opened successfully at 1732184005270 2024-11-21T10:13:25,271 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 90d58495640c0c466573821e63530c34, server=2651e482101a,36083,1732184001229 because future has completed 2024-11-21T10:13:25,272 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d., pid=53, masterSystemTime=1732184005163 2024-11-21T10:13:25,272 INFO [StoreOpener-12bdde3e5981435702484f9c042b20c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:25,273 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=1cda3137e35dc414393695b95490205a, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:25,273 INFO [StoreOpener-b1d292547267d9a1644f4ca59eacbdfa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b1d292547267d9a1644f4ca59eacbdfa columnFamilyName family 2024-11-21T10:13:25,273 DEBUG [StoreOpener-b1d292547267d9a1644f4ca59eacbdfa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,274 INFO [StoreOpener-b1d292547267d9a1644f4ca59eacbdfa-1 {}] regionserver.HStore(327): Store=b1d292547267d9a1644f4ca59eacbdfa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,274 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:25,275 INFO [StoreOpener-12bdde3e5981435702484f9c042b20c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 12bdde3e5981435702484f9c042b20c9 columnFamilyName family 2024-11-21T10:13:25,276 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:25,277 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:25,277 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:25,277 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:25,277 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b19ab03ffba010ed2e3bb6ecb72e84d, NAME => 'test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-21T10:13:25,277 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:25,278 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,278 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,278 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:25,278 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:25,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1cda3137e35dc414393695b95490205a, server=2651e482101a,36761,1732184001491 because future has completed 2024-11-21T10:13:25,278 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:25,279 DEBUG [StoreOpener-12bdde3e5981435702484f9c042b20c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,279 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:25,279 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:25,280 INFO [StoreOpener-12bdde3e5981435702484f9c042b20c9-1 {}] regionserver.HStore(327): Store=12bdde3e5981435702484f9c042b20c9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,280 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:25,281 INFO [StoreOpener-7b19ab03ffba010ed2e3bb6ecb72e84d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:25,282 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:25,283 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:25,284 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:25,284 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:25,284 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:25,285 INFO [StoreOpener-7b19ab03ffba010ed2e3bb6ecb72e84d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b19ab03ffba010ed2e3bb6ecb72e84d columnFamilyName family 2024-11-21T10:13:25,285 DEBUG [StoreOpener-7b19ab03ffba010ed2e3bb6ecb72e84d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,286 INFO [StoreOpener-7b19ab03ffba010ed2e3bb6ecb72e84d-1 {}] regionserver.HStore(327): Store=7b19ab03ffba010ed2e3bb6ecb72e84d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,287 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:25,288 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b1d292547267d9a1644f4ca59eacbdfa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,288 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:25,289 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:25,288 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened b1d292547267d9a1644f4ca59eacbdfa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61070084, jitterRate=-0.08998483419418335}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,289 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:25,289 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for b1d292547267d9a1644f4ca59eacbdfa: Running coprocessor pre-open hook at 1732184005265Writing region info on filesystem at 1732184005265Initializing all the Stores at 1732184005267 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005267Cleaning up temporary data from old regions at 1732184005279 (+12 ms)Running coprocessor post-open hooks at 1732184005289 (+10 ms)Region opened successfully at 1732184005289 2024-11-21T10:13:25,290 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:25,291 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:25,291 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:25,291 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/12bdde3e5981435702484f9c042b20c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,33515,1732184001551 because future has completed 2024-11-21T10:13:25,292 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 12bdde3e5981435702484f9c042b20c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73276922, jitterRate=0.09191122651100159}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,292 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:25,292 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 12bdde3e5981435702484f9c042b20c9: Running coprocessor pre-open hook at 1732184005268Writing region info on filesystem at 1732184005268Initializing all the Stores at 1732184005272 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005272Cleaning up temporary data from old regions at 1732184005284 (+12 ms)Running coprocessor post-open hooks at 1732184005292 (+8 ms)Region opened successfully at 1732184005292 2024-11-21T10:13:25,293 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa., pid=41, masterSystemTime=1732184005115 2024-11-21T10:13:25,293 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2651e482101a,36761,1732184001491, table=test, region=1cda3137e35dc414393695b95490205a. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T10:13:25,294 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2651e482101a,36083,1732184001229, table=test, region=ed68f59ece209cbf633920e19d38f8f5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T10:13:25,294 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2651e482101a,36083,1732184001229, table=test, region=90d58495640c0c466573821e63530c34. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T10:13:25,295 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9., pid=39, masterSystemTime=1732184005059 2024-11-21T10:13:25,295 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2651e482101a,33515,1732184001551, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T10:13:25,296 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:25,298 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:25,298 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:25,298 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:25,298 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => b5794ef6bcf8b6f343b167ad897a2aa0, NAME => 'test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-21T10:13:25,299 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:25,299 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,299 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:25,299 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:25,300 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/7b19ab03ffba010ed2e3bb6ecb72e84d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,301 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:25,301 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:25,301 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 7b19ab03ffba010ed2e3bb6ecb72e84d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68957598, jitterRate=0.027548283338546753}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,301 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:25,301 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:25,301 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 7b19ab03ffba010ed2e3bb6ecb72e84d: Running coprocessor pre-open hook at 1732184005278Writing region info on filesystem at 1732184005278Initializing all the Stores at 1732184005280 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005280Cleaning up temporary data from old regions at 1732184005291 (+11 ms)Running coprocessor post-open hooks at 1732184005301 (+10 ms)Region opened successfully at 1732184005301 2024-11-21T10:13:25,301 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 88fcaf42f6cd2d2a83b7023a1344f960, NAME => 'test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-21T10:13:25,302 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:25,302 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,302 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:25,302 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:25,303 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d., pid=55, masterSystemTime=1732184005163 2024-11-21T10:13:25,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=11 2024-11-21T10:13:25,304 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 1cda3137e35dc414393695b95490205a, server=2651e482101a,36761,1732184001491 in 368 msec 2024-11-21T10:13:25,304 INFO [StoreOpener-b5794ef6bcf8b6f343b167ad897a2aa0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:25,306 INFO [StoreOpener-88fcaf42f6cd2d2a83b7023a1344f960-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:25,306 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:25,306 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:25,307 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003. 2024-11-21T10:13:25,307 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 04a9f41b735b6d4d1061d11055315003, NAME => 'test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-21T10:13:25,307 INFO [StoreOpener-b5794ef6bcf8b6f343b167ad897a2aa0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b5794ef6bcf8b6f343b167ad897a2aa0 columnFamilyName family 2024-11-21T10:13:25,307 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 04a9f41b735b6d4d1061d11055315003 2024-11-21T10:13:25,307 DEBUG [StoreOpener-b5794ef6bcf8b6f343b167ad897a2aa0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,307 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,308 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 04a9f41b735b6d4d1061d11055315003 2024-11-21T10:13:25,308 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 04a9f41b735b6d4d1061d11055315003 2024-11-21T10:13:25,308 INFO [StoreOpener-88fcaf42f6cd2d2a83b7023a1344f960-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88fcaf42f6cd2d2a83b7023a1344f960 columnFamilyName family 2024-11-21T10:13:25,309 INFO [StoreOpener-b5794ef6bcf8b6f343b167ad897a2aa0-1 {}] regionserver.HStore(327): Store=b5794ef6bcf8b6f343b167ad897a2aa0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,309 DEBUG [StoreOpener-88fcaf42f6cd2d2a83b7023a1344f960-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,309 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:25,309 INFO [StoreOpener-88fcaf42f6cd2d2a83b7023a1344f960-1 {}] regionserver.HStore(327): Store=88fcaf42f6cd2d2a83b7023a1344f960/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=1cda3137e35dc414393695b95490205a, ASSIGN in 573 msec 2024-11-21T10:13:25,309 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:25,310 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:25,311 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:25,311 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:25,311 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,312 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:25,312 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:25,312 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:25,312 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:25,312 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:25,314 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:25,316 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,33515,1732184001551 because future has completed 2024-11-21T10:13:25,316 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:25,317 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:25,320 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 12bdde3e5981435702484f9c042b20c9, server=2651e482101a,36761,1732184001491 because future has completed 2024-11-21T10:13:25,321 INFO [StoreOpener-04a9f41b735b6d4d1061d11055315003-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 04a9f41b735b6d4d1061d11055315003 2024-11-21T10:13:25,322 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b5794ef6bcf8b6f343b167ad897a2aa0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,323 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened b5794ef6bcf8b6f343b167ad897a2aa0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70829749, jitterRate=0.055445507168769836}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,323 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:25,323 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for b5794ef6bcf8b6f343b167ad897a2aa0: Running coprocessor pre-open hook at 1732184005299Writing region info on filesystem at 1732184005299Initializing all the Stores at 1732184005301 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005301Cleaning up temporary data from old regions at 1732184005312 (+11 ms)Running coprocessor post-open hooks at 1732184005323 (+11 ms)Region opened successfully at 1732184005323 2024-11-21T10:13:25,323 INFO [StoreOpener-04a9f41b735b6d4d1061d11055315003-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 04a9f41b735b6d4d1061d11055315003 columnFamilyName family 2024-11-21T10:13:25,323 DEBUG [StoreOpener-04a9f41b735b6d4d1061d11055315003-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,324 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0., pid=42, masterSystemTime=1732184005115 2024-11-21T10:13:25,324 INFO [StoreOpener-04a9f41b735b6d4d1061d11055315003-1 {}] regionserver.HStore(327): Store=04a9f41b735b6d4d1061d11055315003/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,325 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/88fcaf42f6cd2d2a83b7023a1344f960/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,325 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 04a9f41b735b6d4d1061d11055315003 2024-11-21T10:13:25,326 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 88fcaf42f6cd2d2a83b7023a1344f960; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64699823, jitterRate=-0.03589750826358795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,326 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:25,326 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/04a9f41b735b6d4d1061d11055315003 2024-11-21T10:13:25,326 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 88fcaf42f6cd2d2a83b7023a1344f960: Running coprocessor pre-open hook at 1732184005302Writing region info on filesystem at 1732184005302Initializing all the Stores at 1732184005304 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005304Cleaning up temporary data from old regions at 1732184005312 (+8 ms)Running coprocessor post-open hooks at 1732184005326 (+14 ms)Region opened successfully at 1732184005326 2024-11-21T10:13:25,327 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/04a9f41b735b6d4d1061d11055315003 2024-11-21T10:13:25,327 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960., pid=34, masterSystemTime=1732184005059 2024-11-21T10:13:25,327 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 04a9f41b735b6d4d1061d11055315003 2024-11-21T10:13:25,327 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 04a9f41b735b6d4d1061d11055315003 2024-11-21T10:13:25,328 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:25,328 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:25,328 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:25,328 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => 34a8113c45096a8a44b690d6f96ba3b3, NAME => 'test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-21T10:13:25,329 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:25,329 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,329 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for 34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:25,329 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for 34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:25,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=8 2024-11-21T10:13:25,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,33515,1732184001551 in 271 msec 2024-11-21T10:13:25,331 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:25,331 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:25,331 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 04a9f41b735b6d4d1061d11055315003 2024-11-21T10:13:25,334 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN in 600 msec 2024-11-21T10:13:25,335 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:25,337 INFO [StoreOpener-34a8113c45096a8a44b690d6f96ba3b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:25,338 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/04a9f41b735b6d4d1061d11055315003/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,339 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88fcaf42f6cd2d2a83b7023a1344f960, server=2651e482101a,36761,1732184001491 because future has completed 2024-11-21T10:13:25,339 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 04a9f41b735b6d4d1061d11055315003; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68051535, jitterRate=0.014046892523765564}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,339 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 04a9f41b735b6d4d1061d11055315003 2024-11-21T10:13:25,339 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 04a9f41b735b6d4d1061d11055315003: Running coprocessor pre-open hook at 1732184005308Writing region info on filesystem at 1732184005308Initializing all the Stores at 1732184005309 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005309Cleaning up temporary data from old regions at 1732184005327 (+18 ms)Running coprocessor post-open hooks at 1732184005339 (+12 ms)Region opened successfully at 1732184005339 2024-11-21T10:13:25,340 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:25,340 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003., pid=56, masterSystemTime=1732184005163 2024-11-21T10:13:25,341 INFO [StoreOpener-34a8113c45096a8a44b690d6f96ba3b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 34a8113c45096a8a44b690d6f96ba3b3 columnFamilyName family 2024-11-21T10:13:25,341 DEBUG [StoreOpener-34a8113c45096a8a44b690d6f96ba3b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,342 INFO [StoreOpener-34a8113c45096a8a44b690d6f96ba3b3-1 {}] regionserver.HStore(327): Store=34a8113c45096a8a44b690d6f96ba3b3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,342 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for 34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:25,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure b5794ef6bcf8b6f343b167ad897a2aa0, server=2651e482101a,36083,1732184001229 because future has completed 2024-11-21T10:13:25,344 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:25,344 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003. 2024-11-21T10:13:25,344 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,iii,1732184003970.04a9f41b735b6d4d1061d11055315003. 2024-11-21T10:13:25,345 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:25,345 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. 2024-11-21T10:13:25,345 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => ab23bbb87a53468249d295336f14f16f, NAME => 'test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-21T10:13:25,345 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for 34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:25,345 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:25,345 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for 34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:25,345 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,345 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:25,345 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:25,347 INFO [StoreOpener-ab23bbb87a53468249d295336f14f16f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:25,348 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for 34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:25,350 INFO [StoreOpener-ab23bbb87a53468249d295336f14f16f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab23bbb87a53468249d295336f14f16f columnFamilyName family 2024-11-21T10:13:25,350 DEBUG [StoreOpener-ab23bbb87a53468249d295336f14f16f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=6 2024-11-21T10:13:25,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 88fcaf42f6cd2d2a83b7023a1344f960, server=2651e482101a,36761,1732184001491 in 422 msec 2024-11-21T10:13:25,351 INFO [StoreOpener-ab23bbb87a53468249d295336f14f16f-1 {}] regionserver.HStore(327): Store=ab23bbb87a53468249d295336f14f16f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,351 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:25,352 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:25,353 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:25,353 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/34a8113c45096a8a44b690d6f96ba3b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,354 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:25,354 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:25,354 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=88fcaf42f6cd2d2a83b7023a1344f960, ASSIGN in 620 msec 2024-11-21T10:13:25,354 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened 34a8113c45096a8a44b690d6f96ba3b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68321978, jitterRate=0.018076807260513306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,354 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:25,354 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for 34a8113c45096a8a44b690d6f96ba3b3: Running coprocessor pre-open hook at 1732184005329Writing region info on filesystem at 1732184005329Initializing all the Stores at 1732184005330 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005330Cleaning up temporary data from old regions at 1732184005345 (+15 ms)Running coprocessor post-open hooks at 1732184005354 (+9 ms)Region opened successfully at 1732184005354 2024-11-21T10:13:25,355 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3., pid=45, masterSystemTime=1732184005115 2024-11-21T10:13:25,357 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:25,358 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:25,358 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:25,359 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:25,359 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 460b55b3159e046531bc058dc9b7c499, NAME => 'test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-21T10:13:25,359 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:25,359 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,359 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:25,359 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:25,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=18 2024-11-21T10:13:25,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure b5794ef6bcf8b6f343b167ad897a2aa0, server=2651e482101a,36083,1732184001229 in 386 msec 2024-11-21T10:13:25,361 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ab23bbb87a53468249d295336f14f16f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,362 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened ab23bbb87a53468249d295336f14f16f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63923388, jitterRate=-0.04746729135513306}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,363 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:25,363 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for ab23bbb87a53468249d295336f14f16f: Running coprocessor pre-open hook at 1732184005346Writing region info on filesystem at 1732184005346Initializing all the Stores at 1732184005347 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005347Cleaning up temporary data from old regions at 1732184005354 (+7 ms)Running coprocessor post-open hooks at 1732184005363 (+9 ms)Region opened successfully at 1732184005363 2024-11-21T10:13:25,362 INFO [StoreOpener-460b55b3159e046531bc058dc9b7c499-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:25,364 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f., pid=47, masterSystemTime=1732184005163 2024-11-21T10:13:25,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b5794ef6bcf8b6f343b167ad897a2aa0, ASSIGN in 631 msec 2024-11-21T10:13:25,366 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:25,366 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. 2024-11-21T10:13:25,367 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. 2024-11-21T10:13:25,367 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3. 2024-11-21T10:13:25,367 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 60a0ac92d2da0863bf4f5f1aac0c8da3, NAME => 'test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-21T10:13:25,367 INFO [StoreOpener-460b55b3159e046531bc058dc9b7c499-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 460b55b3159e046531bc058dc9b7c499 columnFamilyName family 2024-11-21T10:13:25,367 DEBUG [StoreOpener-460b55b3159e046531bc058dc9b7c499-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,367 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 60a0ac92d2da0863bf4f5f1aac0c8da3 2024-11-21T10:13:25,367 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,367 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 60a0ac92d2da0863bf4f5f1aac0c8da3 2024-11-21T10:13:25,368 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 60a0ac92d2da0863bf4f5f1aac0c8da3 2024-11-21T10:13:25,369 INFO [StoreOpener-460b55b3159e046531bc058dc9b7c499-1 {}] regionserver.HStore(327): Store=460b55b3159e046531bc058dc9b7c499/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,369 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:25,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 34a8113c45096a8a44b690d6f96ba3b3, server=2651e482101a,36083,1732184001229 because future has completed 2024-11-21T10:13:25,370 INFO [StoreOpener-60a0ac92d2da0863bf4f5f1aac0c8da3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 60a0ac92d2da0863bf4f5f1aac0c8da3 2024-11-21T10:13:25,371 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:25,371 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:25,372 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=ab23bbb87a53468249d295336f14f16f, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,372 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:25,372 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:25,373 INFO [StoreOpener-60a0ac92d2da0863bf4f5f1aac0c8da3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a0ac92d2da0863bf4f5f1aac0c8da3 columnFamilyName family 2024-11-21T10:13:25,373 DEBUG [StoreOpener-60a0ac92d2da0863bf4f5f1aac0c8da3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,375 INFO [StoreOpener-60a0ac92d2da0863bf4f5f1aac0c8da3-1 {}] regionserver.HStore(327): Store=60a0ac92d2da0863bf4f5f1aac0c8da3/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,375 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 60a0ac92d2da0863bf4f5f1aac0c8da3 2024-11-21T10:13:25,375 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab23bbb87a53468249d295336f14f16f, server=2651e482101a,33515,1732184001551 because future has completed 2024-11-21T10:13:25,375 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:25,376 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/60a0ac92d2da0863bf4f5f1aac0c8da3 2024-11-21T10:13:25,377 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/60a0ac92d2da0863bf4f5f1aac0c8da3 2024-11-21T10:13:25,378 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 60a0ac92d2da0863bf4f5f1aac0c8da3 2024-11-21T10:13:25,378 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 60a0ac92d2da0863bf4f5f1aac0c8da3 2024-11-21T10:13:25,380 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/460b55b3159e046531bc058dc9b7c499/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,381 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 460b55b3159e046531bc058dc9b7c499; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70079141, jitterRate=0.044260576367378235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,381 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:25,381 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 460b55b3159e046531bc058dc9b7c499: Running coprocessor pre-open hook at 1732184005360Writing region info on filesystem at 1732184005360Initializing all the Stores at 1732184005361 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005361Cleaning up temporary data from old regions at 1732184005372 (+11 ms)Running coprocessor post-open hooks at 1732184005381 (+9 ms)Region opened successfully at 1732184005381 2024-11-21T10:13:25,381 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 60a0ac92d2da0863bf4f5f1aac0c8da3 2024-11-21T10:13:25,382 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499., pid=40, masterSystemTime=1732184005115 2024-11-21T10:13:25,384 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/60a0ac92d2da0863bf4f5f1aac0c8da3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=14 2024-11-21T10:13:25,385 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 60a0ac92d2da0863bf4f5f1aac0c8da3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71349056, jitterRate=0.06318378448486328}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 34a8113c45096a8a44b690d6f96ba3b3, server=2651e482101a,36083,1732184001229 in 383 msec 2024-11-21T10:13:25,385 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 60a0ac92d2da0863bf4f5f1aac0c8da3 2024-11-21T10:13:25,385 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 60a0ac92d2da0863bf4f5f1aac0c8da3: Running coprocessor pre-open hook at 1732184005368Writing region info on filesystem at 1732184005368Initializing all the Stores at 1732184005369 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005369Cleaning up temporary data from old regions at 1732184005378 (+9 ms)Running coprocessor post-open hooks at 1732184005385 (+7 ms)Region opened successfully at 1732184005385 2024-11-21T10:13:25,386 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:25,387 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:25,387 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3., pid=54, masterSystemTime=1732184005163 2024-11-21T10:13:25,387 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:25,387 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 51fb43bd943dd75b745159a3a38956c8, NAME => 'test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-21T10:13:25,387 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:25,387 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,388 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:25,388 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:25,389 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3. 2024-11-21T10:13:25,389 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3. 2024-11-21T10:13:25,390 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:25,390 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => fb8bd192054304ea3eed2270c166a773, NAME => 'test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-21T10:13:25,390 INFO [StoreOpener-51fb43bd943dd75b745159a3a38956c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:25,390 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:25,390 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,390 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:25,390 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:25,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=34a8113c45096a8a44b690d6f96ba3b3, ASSIGN in 654 msec 2024-11-21T10:13:25,393 INFO [StoreOpener-51fb43bd943dd75b745159a3a38956c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 51fb43bd943dd75b745159a3a38956c8 columnFamilyName family 2024-11-21T10:13:25,393 INFO [StoreOpener-fb8bd192054304ea3eed2270c166a773-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:25,393 DEBUG [StoreOpener-51fb43bd943dd75b745159a3a38956c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,393 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,394 INFO [StoreOpener-51fb43bd943dd75b745159a3a38956c8-1 {}] regionserver.HStore(327): Store=51fb43bd943dd75b745159a3a38956c8/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,394 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:25,395 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:25,396 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:25,397 INFO [StoreOpener-fb8bd192054304ea3eed2270c166a773-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb8bd192054304ea3eed2270c166a773 columnFamilyName family 2024-11-21T10:13:25,397 DEBUG [StoreOpener-fb8bd192054304ea3eed2270c166a773-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,398 INFO [StoreOpener-fb8bd192054304ea3eed2270c166a773-1 {}] regionserver.HStore(327): Store=fb8bd192054304ea3eed2270c166a773/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,398 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:25,398 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:25,398 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:25,399 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=54, ppid=7, state=RUNNABLE, hasLock=true; OpenRegionProcedure 60a0ac92d2da0863bf4f5f1aac0c8da3, server=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,400 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2651e482101a,36761,1732184001491, table=test, region=12bdde3e5981435702484f9c042b20c9. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T10:13:25,400 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2651e482101a,36083,1732184001229, table=test, region=ed68f59ece209cbf633920e19d38f8f5. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T10:13:25,401 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2651e482101a,36083,1732184001229, table=test, region=90d58495640c0c466573821e63530c34. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T10:13:25,401 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:25,402 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:25,403 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2651e482101a,33515,1732184001551, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T10:13:25,403 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=2651e482101a,33515,1732184001551, table=test, region=ab23bbb87a53468249d295336f14f16f. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-21T10:13:25,403 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:25,403 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:25,404 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:25,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=7 2024-11-21T10:13:25,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 60a0ac92d2da0863bf4f5f1aac0c8da3, server=2651e482101a,33515,1732184001551 in 353 msec 2024-11-21T10:13:25,405 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:25,407 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/51fb43bd943dd75b745159a3a38956c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,408 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 51fb43bd943dd75b745159a3a38956c8; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63995296, jitterRate=-0.04639577865600586}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,408 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:25,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN in 674 msec 2024-11-21T10:13:25,408 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 51fb43bd943dd75b745159a3a38956c8: Running coprocessor pre-open hook at 1732184005388Writing region info on filesystem at 1732184005388Initializing all the Stores at 1732184005390 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005390Cleaning up temporary data from old regions at 1732184005399 (+9 ms)Running coprocessor post-open hooks at 1732184005408 (+9 ms)Region opened successfully at 1732184005408 2024-11-21T10:13:25,409 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/fb8bd192054304ea3eed2270c166a773/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,409 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8., pid=44, masterSystemTime=1732184005115 2024-11-21T10:13:25,410 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:25,410 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened fb8bd192054304ea3eed2270c166a773; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73409395, jitterRate=0.09388522803783417}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,410 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:25,410 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for fb8bd192054304ea3eed2270c166a773: Running coprocessor pre-open hook at 1732184005391Writing region info on filesystem at 1732184005391Initializing all the Stores at 1732184005392 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005392Cleaning up temporary data from old regions at 1732184005404 (+12 ms)Running coprocessor post-open hooks at 1732184005410 (+6 ms)Region opened successfully at 1732184005410 2024-11-21T10:13:25,411 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773., pid=50, masterSystemTime=1732184005163 2024-11-21T10:13:25,413 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:25,413 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:25,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 460b55b3159e046531bc058dc9b7c499, server=2651e482101a,36083,1732184001229 because future has completed 2024-11-21T10:13:25,414 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:25,415 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:25,415 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:25,415 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c. 2024-11-21T10:13:25,416 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 9226917edc2a4eac226de198b54fce9c, NAME => 'test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-21T10:13:25,416 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9226917edc2a4eac226de198b54fce9c 2024-11-21T10:13:25,416 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:25,416 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 9226917edc2a4eac226de198b54fce9c 2024-11-21T10:13:25,416 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 9226917edc2a4eac226de198b54fce9c 2024-11-21T10:13:25,419 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 51fb43bd943dd75b745159a3a38956c8, server=2651e482101a,36083,1732184001229 because future has completed 2024-11-21T10:13:25,420 INFO [StoreOpener-9226917edc2a4eac226de198b54fce9c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9226917edc2a4eac226de198b54fce9c 2024-11-21T10:13:25,422 INFO [StoreOpener-9226917edc2a4eac226de198b54fce9c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9226917edc2a4eac226de198b54fce9c columnFamilyName family 2024-11-21T10:13:25,422 DEBUG [StoreOpener-9226917edc2a4eac226de198b54fce9c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:25,423 INFO [StoreOpener-9226917edc2a4eac226de198b54fce9c-1 {}] regionserver.HStore(327): Store=9226917edc2a4eac226de198b54fce9c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:25,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=30 2024-11-21T10:13:25,423 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 9226917edc2a4eac226de198b54fce9c 2024-11-21T10:13:25,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 460b55b3159e046531bc058dc9b7c499, server=2651e482101a,36083,1732184001229 in 460 msec 2024-11-21T10:13:25,424 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/9226917edc2a4eac226de198b54fce9c 2024-11-21T10:13:25,425 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/9226917edc2a4eac226de198b54fce9c 2024-11-21T10:13:25,425 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 9226917edc2a4eac226de198b54fce9c 2024-11-21T10:13:25,425 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 9226917edc2a4eac226de198b54fce9c 2024-11-21T10:13:25,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=460b55b3159e046531bc058dc9b7c499, ASSIGN in 692 msec 2024-11-21T10:13:25,428 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 9226917edc2a4eac226de198b54fce9c 2024-11-21T10:13:25,431 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/9226917edc2a4eac226de198b54fce9c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-21T10:13:25,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=27 2024-11-21T10:13:25,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 51fb43bd943dd75b745159a3a38956c8, server=2651e482101a,36083,1732184001229 in 445 msec 2024-11-21T10:13:25,432 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 9226917edc2a4eac226de198b54fce9c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63292393, jitterRate=-0.05686984956264496}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:25,432 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9226917edc2a4eac226de198b54fce9c 2024-11-21T10:13:25,433 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 9226917edc2a4eac226de198b54fce9c: Running coprocessor pre-open hook at 1732184005417Writing region info on filesystem at 1732184005417Initializing all the Stores at 1732184005419 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184005419Cleaning up temporary data from old regions at 1732184005425 (+6 ms)Running coprocessor post-open hooks at 1732184005433 (+8 ms)Region opened successfully at 1732184005433 2024-11-21T10:13:25,434 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c., pid=49, masterSystemTime=1732184005163 2024-11-21T10:13:25,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=51fb43bd943dd75b745159a3a38956c8, ASSIGN in 701 msec 2024-11-21T10:13:25,436 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=fb8bd192054304ea3eed2270c166a773, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,438 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c. 2024-11-21T10:13:25,438 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,sss,1732184003970.9226917edc2a4eac226de198b54fce9c. 2024-11-21T10:13:25,439 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,33515,1732184001551 because future has completed 2024-11-21T10:13:25,439 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=9226917edc2a4eac226de198b54fce9c, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,33515,1732184001551 because future has completed 2024-11-21T10:13:25,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=26 2024-11-21T10:13:25,446 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,33515,1732184001551 in 419 msec 2024-11-21T10:13:25,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN in 715 msec 2024-11-21T10:13:25,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=23 2024-11-21T10:13:25,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,33515,1732184001551 in 430 msec 2024-11-21T10:13:25,454 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN in 721 msec 2024-11-21T10:13:25,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=19 2024-11-21T10:13:25,459 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure ab23bbb87a53468249d295336f14f16f, server=2651e482101a,33515,1732184001551 in 447 msec 2024-11-21T10:13:25,461 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN in 728 msec 2024-11-21T10:13:25,462 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=04a9f41b735b6d4d1061d11055315003, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,33515,1732184001551 2024-11-21T10:13:25,465 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,33515,1732184001551 because future has completed 2024-11-21T10:13:25,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=28 2024-11-21T10:13:25,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 12bdde3e5981435702484f9c042b20c9, server=2651e482101a,36761,1732184001491 in 515 msec 2024-11-21T10:13:25,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=12bdde3e5981435702484f9c042b20c9, ASSIGN in 737 msec 2024-11-21T10:13:25,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=13 2024-11-21T10:13:25,473 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,33515,1732184001551 in 417 msec 2024-11-21T10:13:25,475 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN in 743 msec 2024-11-21T10:13:25,476 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, openSeqNum=2, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:25,479 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure b1d292547267d9a1644f4ca59eacbdfa, server=2651e482101a,36083,1732184001229 because future has completed 2024-11-21T10:13:25,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=16 2024-11-21T10:13:25,481 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,33515,1732184001551 in 439 msec 2024-11-21T10:13:25,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN in 750 msec 2024-11-21T10:13:25,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=21 2024-11-21T10:13:25,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure b1d292547267d9a1644f4ca59eacbdfa, server=2651e482101a,36083,1732184001229 in 518 msec 2024-11-21T10:13:25,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b1d292547267d9a1644f4ca59eacbdfa, ASSIGN in 756 msec 2024-11-21T10:13:25,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=12 2024-11-21T10:13:25,491 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 90d58495640c0c466573821e63530c34, server=2651e482101a,36083,1732184001229 in 457 msec 2024-11-21T10:13:25,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=90d58495640c0c466573821e63530c34, ASSIGN in 760 msec 2024-11-21T10:13:25,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=5 2024-11-21T10:13:25,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure ed68f59ece209cbf633920e19d38f8f5, server=2651e482101a,36083,1732184001229 in 493 msec 2024-11-21T10:13:25,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-21T10:13:25,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ed68f59ece209cbf633920e19d38f8f5, ASSIGN in 766 msec 2024-11-21T10:13:25,501 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-21T10:13:25,501 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732184005501"}]},"ts":"1732184005501"} 2024-11-21T10:13:25,506 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-21T10:13:25,509 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-21T10:13:25,516 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.5310 sec 2024-11-21T10:13:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-21T10:13:26,146 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-21T10:13:26,146 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-21T10:13:26,149 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T10:13:26,173 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-21T10:13:26,174 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T10:13:26,175 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-21T10:13:26,175 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T10:13:26,182 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-21T10:13:26,206 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-21T10:13:26,222 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T10:13:26,225 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60002, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T10:13:26,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T10:13:26,230 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56018, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T10:13:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 60a0ac92d2da0863bf4f5f1aac0c8da3 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7b19ab03ffba010ed2e3bb6ecb72e84d changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 04a9f41b735b6d4d1061d11055315003 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f74cd0d96e89c26d17d931ba8b44a79d changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ab23bbb87a53468249d295336f14f16f changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 32979bec7fcef8d630309c08ee22a6e6 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9226917edc2a4eac226de198b54fce9c changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fb8bd192054304ea3eed2270c166a773 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ed68f59ece209cbf633920e19d38f8f5 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0fbe63f3c83c91920dd0c08974ab5ea8 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 90d58495640c0c466573821e63530c34 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 34a8113c45096a8a44b690d6f96ba3b3 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b5794ef6bcf8b6f343b167ad897a2aa0 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b1d292547267d9a1644f4ca59eacbdfa changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ba34ea0b7addcd538aef55c8823e35dc changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 51fb43bd943dd75b745159a3a38956c8 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 460b55b3159e046531bc058dc9b7c499 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 88fcaf42f6cd2d2a83b7023a1344f960 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8b2fc4f9ae6f60de0b3c78d355788c60 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1cda3137e35dc414393695b95490205a changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 52a49023e46e4cdb48a9866360492959 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b207ca8401450ab08698a420b743f234 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region d0e3296740e89bbfb20e449acd71bdd4 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e90dd76cb8307eb049be31deaec171dd changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 12bdde3e5981435702484f9c042b20c9 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 019a4c4ad79900668e68f2029ed4ec91 changed from -1.0 to 0.0, refreshing cache 2024-11-21T10:13:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-21T10:13:26,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-21T10:13:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-21T10:13:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BalancerClusterState(204): Hosts are {2651e482101a=0} racks are {/default-rack=0} 2024-11-21T10:13:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-21T10:13:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-21T10:13:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-21T10:13:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T10:13:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T10:13:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-21T10:13:26,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T10:13:26,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T10:13:26,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-21T10:13:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-21T10:13:26,308 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-21T10:13:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-21T10:13:26,311 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T10:13:26,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.2 set balanceSwitch=false 2024-11-21T10:13:26,344 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=385, OpenFileDescriptor=651, MaxFileDescriptor=1048576, SystemLoadAverage=368, ProcessCount=11, AvailableMemoryMB=2717 2024-11-21T10:13:26,348 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-21T10:13:26,349 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-21T10:13:26,349 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 2651e482101a,33515,1732184001551 2024-11-21T10:13:26,350 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 2651e482101a,33515,1732184001551: Simulated kill ***** 2024-11-21T10:13:26,350 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-21T10:13:26,352 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-21T10:13:26,355 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-21T10:13:26,356 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-21T10:13:26,356 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-21T10:13:26,356 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 311167064 }, "NonHeapMemoryUsage": { "committed": 151126016, "init": 7667712, "max": -1, "used": 148864880 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "2651e482101a" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "2651e482101a" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "2651e482101a" } ] } 2024-11-21T10:13:26,360 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35251 {}] master.MasterRpcServices(700): 2651e482101a,33515,1732184001551 reported a fatal error: ***** ABORTING region server 2651e482101a,33515,1732184001551: Simulated kill ***** 2024-11-21T10:13:26,363 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2651e482101a,33515,1732184001551' ***** 2024-11-21T10:13:26,363 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-21T10:13:26,363 INFO [RS:2;2651e482101a:33515 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T10:13:26,364 INFO [RS:2;2651e482101a:33515 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-21T10:13:26,364 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T10:13:26,364 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T10:13:26,364 INFO [RS:2;2651e482101a:33515 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-21T10:13:26,364 INFO [RS:2;2651e482101a:33515 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T10:13:26,364 INFO [RS:2;2651e482101a:33515 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;2651e482101a:33515. 2024-11-21T10:13:26,364 DEBUG [RS:2;2651e482101a:33515 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T10:13:26,369 DEBUG [RS:2;2651e482101a:33515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T10:13:26,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741836_1012 (size=32114) 2024-11-21T10:13:26,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741836_1012 (size=32114) 2024-11-21T10:13:26,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741836_1012 (size=32114) 2024-11-21T10:13:26,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741835_1011 (size=2237) 2024-11-21T10:13:26,378 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741835_1011 (size=2237) 2024-11-21T10:13:26,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741835_1011 (size=2237) 2024-11-21T10:13:26,379 DEBUG [RS:2;2651e482101a:33515 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T10:13:26,379 INFO [RS:2;2651e482101a:33515 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T10:13:26,380 INFO [RS:2;2651e482101a:33515 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33515 2024-11-21T10:13:26,387 INFO [regionserver/2651e482101a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T10:13:26,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2651e482101a,33515,1732184001551 2024-11-21T10:13:26,451 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T10:13:26,451 INFO [RS:2;2651e482101a:33515 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T10:13:26,464 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2651e482101a,33515,1732184001551] 2024-11-21T10:13:26,471 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2651e482101a,33515,1732184001551 already deleted, retry=false 2024-11-21T10:13:26,472 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 2651e482101a,33515,1732184001551 on 2651e482101a,35251,1732184000477 2024-11-21T10:13:26,476 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:26,480 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=57 for 2651e482101a,33515,1732184001551 (carryingMeta=true) 2651e482101a,33515,1732184001551/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@73c18fb1[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-21T10:13:26,480 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:26,482 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true, isMeta: true 2024-11-21T10:13:26,484 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting 2024-11-21T10:13:26,486 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2651e482101a,33515,1732184001551 WAL count=1, meta=true 2024-11-21T10:13:26,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta}] 2024-11-21T10:13:26,495 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:26,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:26,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T10:13:26,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33515-0x1015ce240570003, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T10:13:26,562 INFO [RS:2;2651e482101a:33515 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T10:13:26,562 INFO [RS:2;2651e482101a:33515 {}] regionserver.HRegionServer(1031): Exiting; stopping=2651e482101a,33515,1732184001551; zookeeper connection closed. 2024-11-21T10:13:26,563 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28e7f76c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28e7f76c 2024-11-21T10:13:26,565 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-21T10:13:26,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36761 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-21T10:13:26,691 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, size=31.4 K (32114bytes) 2024-11-21T10:13:26,691 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-21T10:13:26,692 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-21T10:13:26,692 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta 2024-11-21T10:13:26,697 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta after 4ms 2024-11-21T10:13:26,704 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta took 15ms 2024-11-21T10:13:26,713 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-21T10:13:26,717 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta.temp 2024-11-21T10:13:26,720 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta.temp 2024-11-21T10:13:26,726 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta so closing down 2024-11-21T10:13:26,726 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T10:13:26,727 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T10:13:26,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741864_1040 (size=31873) 2024-11-21T10:13:26,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741864_1040 (size=31873) 2024-11-21T10:13:26,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741864_1040 (size=31873) 2024-11-21T10:13:26,737 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) 2024-11-21T10:13:26,740 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta.temp to hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-21T10:13:26,745 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 34 ms; skipped=1; WAL=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-21T10:13:26,745 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, journal: Splitting hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, size=31.4 K (32114bytes) at 1732184006691Creating recovered edits writer path=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta.temp at 1732184006721 (+30 ms)Finishing writing output for hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta so closing down at 1732184006726 (+5 ms)3 split writer threads finished at 1732184006727 (+1 ms)Closed recovered edits writer path=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta.temp (wrote 57 edits, skipped 0 edits in 3 ms) at 1732184006737 (+10 ms)Rename recovered edits hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta.temp to hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1732184006740 (+3 ms)Processed 58 edits across 1 Regions in 34 ms; skipped=1; WAL=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1732184006745 (+5 ms) 2024-11-21T10:13:26,746 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta 2024-11-21T10:13:26,747 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-21T10:13:26,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-21T10:13:26,758 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta to hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/oldWALs 2024-11-21T10:13:26,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-21T10:13:26,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 in 261 msec 2024-11-21T10:13:26,763 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:26,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-21T10:13:26,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 in 277 msec 2024-11-21T10:13:26,771 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting dir is empty, no logs to split. 2024-11-21T10:13:26,771 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2651e482101a,33515,1732184001551 WAL count=0, meta=true 2024-11-21T10:13:26,771 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 2651e482101a,33515,1732184001551 WAL splitting is done? wals=0, meta=true 2024-11-21T10:13:26,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-21T10:13:26,776 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:26,777 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T10:13:26,927 DEBUG [2651e482101a:35251 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-21T10:13:26,928 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(204): Hosts are {2651e482101a=0} racks are {/default-rack=0} 2024-11-21T10:13:26,928 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T10:13:26,928 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T10:13:26,928 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T10:13:26,928 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T10:13:26,928 INFO [2651e482101a:35251 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T10:13:26,929 INFO [2651e482101a:35251 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T10:13:26,929 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T10:13:26,929 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:26,931 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPENING 2024-11-21T10:13:26,984 INFO [regionserver/2651e482101a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T10:13:26,985 INFO [regionserver/2651e482101a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T10:13:26,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:26,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:26,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:26,998 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:26,998 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:26,998 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:26,998 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:26,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:27,158 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-21T10:13:27,158 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-21T10:13:27,158 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-21T10:13:27,161 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2651e482101a%2C36761%2C1732184001491.meta, suffix=.meta, logDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,36761,1732184001491, archiveDir=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/oldWALs, maxLogs=32 2024-11-21T10:13:27,176 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,36761,1732184001491/2651e482101a%2C36761%2C1732184001491.meta.1732184007163.meta, exclude list is [], retry=0 2024-11-21T10:13:27,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39431,DS-3d33f7b2-5674-4258-8003-1641935bc33a,DISK] 2024-11-21T10:13:27,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45815,DS-ebed3a63-7f40-4df1-bc25-cc54b1ace770,DISK] 2024-11-21T10:13:27,180 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37847,DS-5d8f2e7a-cf62-4175-abda-564b3f52e773,DISK] 2024-11-21T10:13:27,183 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,36761,1732184001491/2651e482101a%2C36761%2C1732184001491.meta.1732184007163.meta 2024-11-21T10:13:27,184 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44629:44629),(127.0.0.1/127.0.0.1:33461:33461),(127.0.0.1/127.0.0.1:33871:33871)] 2024-11-21T10:13:27,184 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-21T10:13:27,184 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-21T10:13:27,184 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-21T10:13:27,185 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-21T10:13:27,185 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-21T10:13:27,185 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:27,185 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-21T10:13:27,185 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-21T10:13:27,187 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-21T10:13:27,188 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-21T10:13:27,188 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:27,189 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:27,189 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-21T10:13:27,190 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-21T10:13:27,190 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:27,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:27,191 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-21T10:13:27,192 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-21T10:13:27,192 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:27,193 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:27,193 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-21T10:13:27,194 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-21T10:13:27,194 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:27,195 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-21T10:13:27,195 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-21T10:13:27,197 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740 2024-11-21T10:13:27,200 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740 2024-11-21T10:13:27,201 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-21T10:13:27,219 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-21T10:13:27,223 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-21T10:13:27,303 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/info/03a31adc6f04422481a120b1ada40694 is 138, key is test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960./info:regioninfo/1732184005335/Put/seqid=0 2024-11-21T10:13:27,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741866_1042 (size=34924) 2024-11-21T10:13:27,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741866_1042 (size=34924) 2024-11-21T10:13:27,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741866_1042 (size=34924) 2024-11-21T10:13:27,316 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/info/03a31adc6f04422481a120b1ada40694 2024-11-21T10:13:27,380 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/ns/cd33a6c9136a46b49e6aa3f76ce7f0bf is 43, key is default/ns:d/1732184003720/Put/seqid=0 2024-11-21T10:13:27,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741867_1043 (size=5153) 2024-11-21T10:13:27,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741867_1043 (size=5153) 2024-11-21T10:13:27,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741867_1043 (size=5153) 2024-11-21T10:13:27,393 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/ns/cd33a6c9136a46b49e6aa3f76ce7f0bf 2024-11-21T10:13:27,425 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/table/688f3af564764baa9145df72d8b262a7 is 40, key is test/table:state/1732184005501/Put/seqid=0 2024-11-21T10:13:27,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741868_1044 (size=5165) 2024-11-21T10:13:27,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741868_1044 (size=5165) 2024-11-21T10:13:27,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741868_1044 (size=5165) 2024-11-21T10:13:27,837 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/table/688f3af564764baa9145df72d8b262a7 2024-11-21T10:13:27,849 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/info/03a31adc6f04422481a120b1ada40694 as hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/info/03a31adc6f04422481a120b1ada40694 2024-11-21T10:13:27,859 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/info/03a31adc6f04422481a120b1ada40694, entries=260, sequenceid=60, filesize=34.1 K 2024-11-21T10:13:27,862 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/ns/cd33a6c9136a46b49e6aa3f76ce7f0bf as hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/ns/cd33a6c9136a46b49e6aa3f76ce7f0bf 2024-11-21T10:13:27,871 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/ns/cd33a6c9136a46b49e6aa3f76ce7f0bf, entries=2, sequenceid=60, filesize=5.0 K 2024-11-21T10:13:27,873 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/table/688f3af564764baa9145df72d8b262a7 as hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/table/688f3af564764baa9145df72d8b262a7 2024-11-21T10:13:27,881 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/table/688f3af564764baa9145df72d8b262a7, entries=2, sequenceid=60, filesize=5.0 K 2024-11-21T10:13:27,885 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 661ms, sequenceid=60, compaction requested=false; wal=null 2024-11-21T10:13:27,886 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-21T10:13:27,888 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-21T10:13:27,890 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-21T10:13:27,890 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-21T10:13:27,891 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-21T10:13:27,893 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-21T10:13:27,929 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-21T10:13:27,930 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73607714, jitterRate=0.09684041142463684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-21T10:13:27,930 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-21T10:13:27,931 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732184007185Writing region info on filesystem at 1732184007185Initializing all the Stores at 1732184007186 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732184007187 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732184007187Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184007187Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732184007187Obtaining lock to block concurrent updates at 1732184007223 (+36 ms)Preparing flush snapshotting stores in 1588230740 at 1732184007223Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1732184007227 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732184007227Flushing 1588230740/info: creating writer at 1732184007229 (+2 ms)Flushing 1588230740/info: appending metadata at 1732184007287 (+58 ms)Flushing 1588230740/info: closing flushed file at 1732184007290 (+3 ms)Flushing 1588230740/ns: creating writer at 1732184007358 (+68 ms)Flushing 1588230740/ns: appending metadata at 1732184007378 (+20 ms)Flushing 1588230740/ns: closing flushed file at 1732184007378Flushing 1588230740/table: creating writer at 1732184007402 (+24 ms)Flushing 1588230740/table: appending metadata at 1732184007424 (+22 ms)Flushing 1588230740/table: closing flushed file at 1732184007424Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@470d7bc1: reopening flushed file at 1732184007847 (+423 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@669c8dc3: reopening flushed file at 1732184007861 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7804079f: reopening flushed file at 1732184007871 (+10 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 661ms, sequenceid=60, compaction requested=false; wal=null at 1732184007885 (+14 ms)Cleaning up temporary data from old regions at 1732184007890 (+5 ms)Running coprocessor post-open hooks at 1732184007930 (+40 ms)Region opened successfully at 1732184007930 2024-11-21T10:13:27,932 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1732184007152 2024-11-21T10:13:27,935 DEBUG [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-21T10:13:27,935 INFO [RS_OPEN_META-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-21T10:13:27,935 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:27,936 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:27,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:27,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:27,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:27,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:27,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:27,983 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:27,983 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:27,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-21T10:13:27,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 in 985 msec 2024-11-21T10:13:27,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-21T10:13:27,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.2130 sec 2024-11-21T10:13:27,990 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 2651e482101a,33515,1732184001551 had 9 regions 2024-11-21T10:13:27,992 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true, isMeta: false 2024-11-21T10:13:27,995 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2651e482101a,33515,1732184001551 WAL count=1, meta=false 2024-11-21T10:13:27,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140}] 2024-11-21T10:13:27,997 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:27,998 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:28,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36761 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-21T10:13:28,168 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140, size=2.2 K (2237bytes) 2024-11-21T10:13:28,168 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140 2024-11-21T10:13:28,169 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140 after 0ms 2024-11-21T10:13:28,172 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140 took 5ms 2024-11-21T10:13:28,175 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 32979bec7fcef8d630309c08ee22a6e6: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T10:13:28,177 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for f74cd0d96e89c26d17d931ba8b44a79d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T10:13:28,180 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 7b19ab03ffba010ed2e3bb6ecb72e84d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T10:13:28,182 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 04a9f41b735b6d4d1061d11055315003: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T10:13:28,184 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for ab23bbb87a53468249d295336f14f16f: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T10:13:28,185 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 60a0ac92d2da0863bf4f5f1aac0c8da3: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T10:13:28,187 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for fb8bd192054304ea3eed2270c166a773: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T10:13:28,189 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 9226917edc2a4eac226de198b54fce9c: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-21T10:13:28,190 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140 so closing down 2024-11-21T10:13:28,190 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-21T10:13:28,190 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-21T10:13:28,190 INFO [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-21T10:13:28,190 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140, journal: Splitting hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140, size=2.2 K (2237bytes) at 1732184008168Finishing writing output for hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140 so closing down at 1732184008190 (+22 ms)3 split writer threads finished at 1732184008190Processed 8 edits across 0 Regions in 18 ms; skipped=8; WAL=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1732184008190 2024-11-21T10:13:28,190 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140 2024-11-21T10:13:28,190 DEBUG [RS_LOG_REPLAY_OPS-regionserver/2651e482101a:0-1 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-21T10:13:28,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35251 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-21T10:13:28,195 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting/2651e482101a%2C33515%2C1732184001551.1732184003140 to hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/oldWALs 2024-11-21T10:13:28,197 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-21T10:13:28,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 in 197 msec 2024-11-21T10:13:28,199 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,202 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-21T10:13:28,203 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 in 204 msec 2024-11-21T10:13:28,205 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/WALs/2651e482101a,33515,1732184001551-splitting dir is empty, no logs to split. 2024-11-21T10:13:28,205 INFO [PEWorker-1 {}] master.SplitWALManager(105): 2651e482101a,33515,1732184001551 WAL count=0, meta=false 2024-11-21T10:13:28,205 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 2651e482101a,33515,1732184001551 WAL splitting is done? wals=0, meta=false 2024-11-21T10:13:28,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN}] 2024-11-21T10:13:28,210 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:28,211 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T10:13:28,212 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:28,213 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T10:13:28,214 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:28,215 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T10:13:28,216 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:28,217 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T10:13:28,218 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:28,219 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T10:13:28,220 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:28,221 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T10:13:28,222 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:28,223 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T10:13:28,224 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:28,225 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-21T10:13:28,362 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(204): Hosts are {2651e482101a=0} racks are {/default-rack=0} 2024-11-21T10:13:28,362 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-21T10:13:28,362 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-21T10:13:28,362 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-21T10:13:28,362 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-21T10:13:28,362 INFO [2651e482101a:35251 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-21T10:13:28,362 INFO [2651e482101a:35251 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-21T10:13:28,363 DEBUG [2651e482101a:35251 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-21T10:13:28,363 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,364 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,365 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,365 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2651e482101a:33515 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2651e482101a/172.17.0.2:33515 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T10:13:28,366 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, regionLocation=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,366 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,367 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1, error=java.net.ConnectException: Call to address=2651e482101a:33515 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2651e482101a/172.17.0.2:33515 2024-11-21T10:13:28,367 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-21T10:13:28,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1 from cache 2024-11-21T10:13:28,368 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, regionLocation=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,368 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-21T10:13:28,368 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2651e482101a,36761,1732184001491, seqNum=-1] 2024-11-21T10:13:28,368 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-21T10:13:28,369 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 2651e482101a:33515 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2651e482101a/172.17.0.2:33515 2024-11-21T10:13:28,370 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 2651e482101a:33515 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2651e482101a/172.17.0.2:33515 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T10:13:28,370 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42709, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-21T10:13:28,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2651e482101a,36761,1732184001491, seqNum=-1, error=java.net.ConnectException: Call to address=2651e482101a:33515 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2651e482101a/172.17.0.2:33515 2024-11-21T10:13:28,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2651e482101a,36761,1732184001491, seqNum=-1, error=java.net.ConnectException: Call to address=2651e482101a:33515 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2651e482101a/172.17.0.2:33515 2024-11-21T10:13:28,370 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2651e482101a,36761,1732184001491, seqNum=-1, error=java.net.ConnectException: Call to address=2651e482101a:33515 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2651e482101a/172.17.0.2:33515 2024-11-21T10:13:28,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2651e482101a,36761,1732184001491, seqNum=-1, error=java.net.ConnectException: Call to address=2651e482101a:33515 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2651e482101a/172.17.0.2:33515 2024-11-21T10:13:28,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2651e482101a,36761,1732184001491, seqNum=-1, error=java.net.ConnectException: Call to address=2651e482101a:33515 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2651e482101a/172.17.0.2:33515 2024-11-21T10:13:28,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=2651e482101a,33515,1732184001551, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=2651e482101a,36761,1732184001491, seqNum=-1, error=java.net.ConnectException: Call to address=2651e482101a:33515 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2651e482101a/172.17.0.2:33515 2024-11-21T10:13:28,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 2651e482101a:33515 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 2651e482101a/172.17.0.2:33515 2024-11-21T10:13:28,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN because future has completed 2024-11-21T10:13:28,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:28,478 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN because future has completed 2024-11-21T10:13:28,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:28,480 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN because future has completed 2024-11-21T10:13:28,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN because future has completed 2024-11-21T10:13:28,488 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:28,488 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN because future has completed 2024-11-21T10:13:28,489 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN because future has completed 2024-11-21T10:13:28,493 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN because future has completed 2024-11-21T10:13:28,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN because future has completed 2024-11-21T10:13:28,499 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:28,508 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:28,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,36083,1732184001229}] 2024-11-21T10:13:28,526 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a0ac92d2da0863bf4f5f1aac0c8da3, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:28,532 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:28,533 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 7b19ab03ffba010ed2e3bb6ecb72e84d, NAME => 'test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-21T10:13:28,533 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:28,533 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:28,534 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:28,534 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:28,537 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab23bbb87a53468249d295336f14f16f, server=2651e482101a,36761,1732184001491}] 2024-11-21T10:13:28,537 INFO [StoreOpener-7b19ab03ffba010ed2e3bb6ecb72e84d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:28,539 INFO [StoreOpener-7b19ab03ffba010ed2e3bb6ecb72e84d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7b19ab03ffba010ed2e3bb6ecb72e84d columnFamilyName family 2024-11-21T10:13:28,539 DEBUG [StoreOpener-7b19ab03ffba010ed2e3bb6ecb72e84d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:28,540 INFO [StoreOpener-7b19ab03ffba010ed2e3bb6ecb72e84d-1 {}] regionserver.HStore(327): Store=7b19ab03ffba010ed2e3bb6ecb72e84d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:28,540 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:28,542 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:28,546 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:28,548 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:28,548 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:28,553 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:28,555 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 7b19ab03ffba010ed2e3bb6ecb72e84d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67664716, jitterRate=0.008282840251922607}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:28,555 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:28,555 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 7b19ab03ffba010ed2e3bb6ecb72e84d: Running coprocessor pre-open hook at 1732184008534Writing region info on filesystem at 1732184008534Initializing all the Stores at 1732184008536 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184008536Cleaning up temporary data from old regions at 1732184008548 (+12 ms)Running coprocessor post-open hooks at 1732184008555 (+7 ms)Region opened successfully at 1732184008555 2024-11-21T10:13:28,556 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d., pid=72, masterSystemTime=1732184008525 2024-11-21T10:13:28,561 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:28,561 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:28,562 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:28,562 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 32979bec7fcef8d630309c08ee22a6e6, NAME => 'test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-21T10:13:28,562 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:28,563 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:28,563 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:28,563 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:28,578 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-21T10:13:28,578 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,578 INFO [StoreOpener-32979bec7fcef8d630309c08ee22a6e6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:28,578 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:28,580 INFO [StoreOpener-32979bec7fcef8d630309c08ee22a6e6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 32979bec7fcef8d630309c08ee22a6e6 columnFamilyName family 2024-11-21T10:13:28,580 DEBUG [StoreOpener-32979bec7fcef8d630309c08ee22a6e6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:28,581 INFO [StoreOpener-32979bec7fcef8d630309c08ee22a6e6-1 {}] regionserver.HStore(327): Store=32979bec7fcef8d630309c08ee22a6e6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:28,581 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:28,583 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:28,584 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:28,585 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:28,585 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:28,588 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:28,589 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 32979bec7fcef8d630309c08ee22a6e6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66802695, jitterRate=-0.004562273621559143}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:28,590 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:28,590 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 32979bec7fcef8d630309c08ee22a6e6: Running coprocessor pre-open hook at 1732184008564Writing region info on filesystem at 1732184008564Initializing all the Stores at 1732184008572 (+8 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184008572Cleaning up temporary data from old regions at 1732184008585 (+13 ms)Running coprocessor post-open hooks at 1732184008590 (+5 ms)Region opened successfully at 1732184008590 2024-11-21T10:13:28,591 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6., pid=76, masterSystemTime=1732184008525 2024-11-21T10:13:28,597 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:28,598 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:28,598 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:28,598 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => fb8bd192054304ea3eed2270c166a773, NAME => 'test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-21T10:13:28,598 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:28,599 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:28,597 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-21T10:13:28,599 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:28,599 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:28,599 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:28,599 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab23bbb87a53468249d295336f14f16f, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:28,604 INFO [StoreOpener-fb8bd192054304ea3eed2270c166a773-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:28,607 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab23bbb87a53468249d295336f14f16f, server=2651e482101a,36761,1732184001491 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:28,608 INFO [StoreOpener-fb8bd192054304ea3eed2270c166a773-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fb8bd192054304ea3eed2270c166a773 columnFamilyName family 2024-11-21T10:13:28,608 DEBUG [StoreOpener-fb8bd192054304ea3eed2270c166a773-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:28,612 INFO [StoreOpener-fb8bd192054304ea3eed2270c166a773-1 {}] regionserver.HStore(327): Store=fb8bd192054304ea3eed2270c166a773/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:28,612 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:28,613 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:28,615 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:28,616 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:28,616 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:28,621 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:28,622 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened fb8bd192054304ea3eed2270c166a773; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66975678, jitterRate=-0.001984626054763794}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:28,623 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:28,623 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for fb8bd192054304ea3eed2270c166a773: Running coprocessor pre-open hook at 1732184008599Writing region info on filesystem at 1732184008599Initializing all the Stores at 1732184008601 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184008601Cleaning up temporary data from old regions at 1732184008616 (+15 ms)Running coprocessor post-open hooks at 1732184008623 (+7 ms)Region opened successfully at 1732184008623 2024-11-21T10:13:28,624 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773., pid=77, masterSystemTime=1732184008525 2024-11-21T10:13:28,631 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:28,631 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:28,631 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:28,632 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => f74cd0d96e89c26d17d931ba8b44a79d, NAME => 'test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-21T10:13:28,632 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:28,632 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:28,633 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:28,633 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:28,636 INFO [StoreOpener-f74cd0d96e89c26d17d931ba8b44a79d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:28,640 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. 2024-11-21T10:13:28,640 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => ab23bbb87a53468249d295336f14f16f, NAME => 'test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-21T10:13:28,640 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:28,641 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-21T10:13:28,641 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:28,641 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:28,642 INFO [StoreOpener-f74cd0d96e89c26d17d931ba8b44a79d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f74cd0d96e89c26d17d931ba8b44a79d columnFamilyName family 2024-11-21T10:13:28,642 DEBUG [StoreOpener-f74cd0d96e89c26d17d931ba8b44a79d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:28,643 INFO [StoreOpener-f74cd0d96e89c26d17d931ba8b44a79d-1 {}] regionserver.HStore(327): Store=f74cd0d96e89c26d17d931ba8b44a79d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:28,643 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:28,644 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:28,647 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:28,648 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:28,648 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:28,651 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:28,654 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened f74cd0d96e89c26d17d931ba8b44a79d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68325419, jitterRate=0.018128082156181335}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:28,654 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:28,655 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for f74cd0d96e89c26d17d931ba8b44a79d: Running coprocessor pre-open hook at 1732184008633Writing region info on filesystem at 1732184008633Initializing all the Stores at 1732184008634 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184008635 (+1 ms)Cleaning up temporary data from old regions at 1732184008648 (+13 ms)Running coprocessor post-open hooks at 1732184008655 (+7 ms)Region opened successfully at 1732184008655 2024-11-21T10:13:28,655 INFO [StoreOpener-ab23bbb87a53468249d295336f14f16f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:28,657 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d., pid=75, masterSystemTime=1732184008525 2024-11-21T10:13:28,659 INFO [StoreOpener-ab23bbb87a53468249d295336f14f16f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ab23bbb87a53468249d295336f14f16f columnFamilyName family 2024-11-21T10:13:28,659 DEBUG [StoreOpener-ab23bbb87a53468249d295336f14f16f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-21T10:13:28,663 INFO [StoreOpener-ab23bbb87a53468249d295336f14f16f-1 {}] regionserver.HStore(327): Store=ab23bbb87a53468249d295336f14f16f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-21T10:13:28,663 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:28,664 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:28,664 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:28,664 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:28,666 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:28,667 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:28,667 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:28,670 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:28,670 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:28,670 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:28,671 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:28,672 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:28,672 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:28,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:28,673 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:28,674 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:28,674 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:28,674 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:28,676 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened ab23bbb87a53468249d295336f14f16f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64797713, jitterRate=-0.034438833594322205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-21T10:13:28,676 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ab23bbb87a53468249d295336f14f16f 2024-11-21T10:13:28,676 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for ab23bbb87a53468249d295336f14f16f: Running coprocessor pre-open hook at 1732184008641Writing region info on filesystem at 1732184008641Initializing all the Stores at 1732184008642 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732184008642Cleaning up temporary data from old regions at 1732184008667 (+25 ms)Running coprocessor post-open hooks at 1732184008676 (+9 ms)Region opened successfully at 1732184008676 2024-11-21T10:13:28,678 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:28,679 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f., pid=79, masterSystemTime=1732184008633 2024-11-21T10:13:28,680 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:28,685 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:28,686 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,687 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:28,688 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:28,708 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "2651e482101a" port: 36761 start_code: 1732184001491 } transition { transition_code: OPENED region_info { region_id: 1732184003970 table_name { namespace: "default" qualifier: "test" } start_key: "ooo" end_key: "ppp" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1732184001663 }; retry (#0) after 1006ms delay (Master is coming online...). org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1857) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-21T10:13:28,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:28,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:28,731 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a0ac92d2da0863bf4f5f1aac0c8da3, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab23bbb87a53468249d295336f14f16f, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,732 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,733 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,733 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,733 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,733 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,733 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,733 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:28,733 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,733 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:28,733 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,733 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:28,733 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,733 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:28,733 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,733 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:28,733 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,733 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:28,734 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,734 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:28,734 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,734 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:28,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a0ac92d2da0863bf4f5f1aac0c8da3, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,734 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure ab23bbb87a53468249d295336f14f16f, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,735 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 61 msec 2024-11-21T10:13:28,754 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:28,754 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:28,755 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:28,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:28,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:28,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:28,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:28,799 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:28,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:28,801 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:28,801 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:28,801 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:28,801 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:28,801 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:28,801 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN to state=OFFLINE, location=null, table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3 to restore RIT 2024-11-21T10:13:28,801 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN to state=OFFLINE, location=null, table=test, region=9226917edc2a4eac226de198b54fce9c to restore RIT 2024-11-21T10:13:28,801 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN to state=OFFLINE, location=null, table=test, region=ab23bbb87a53468249d295336f14f16f to restore RIT 2024-11-21T10:13:28,821 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184038821 2024-11-21T10:13:28,821 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:28,822 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:28,826 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:28,830 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:28,830 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a0ac92d2da0863bf4f5f1aac0c8da3, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:28,830 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a0ac92d2da0863bf4f5f1aac0c8da3, server=2651e482101a,36761,1732184001491 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:28,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,870 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,871 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,872 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,873 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,874 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,875 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,879 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:28,879 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184068879 2024-11-21T10:13:28,879 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184128879 2024-11-21T10:13:28,879 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 58 msec 2024-11-21T10:13:28,879 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:28,879 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:28,879 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:28,879 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:28,880 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:28,880 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:28,880 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:28,881 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:28,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T10:13:28,892 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:28,893 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:28,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:28,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:28,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:28,894 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:28,894 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:28,899 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:28,917 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:28,917 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:28,918 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:28,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:28,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:28,919 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:28,920 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a0ac92d2da0863bf4f5f1aac0c8da3, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:28,921 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,922 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:28,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:28,922 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,922 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,923 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,923 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,923 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,923 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:28,923 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,923 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a0ac92d2da0863bf4f5f1aac0c8da3, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:28,924 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 41 msec 2024-11-21T10:13:28,926 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:28,926 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:28,926 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:28,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:28,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:28,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:28,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:28,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:28,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:28,936 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:28,937 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:28,937 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:28,937 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:28,937 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:28,937 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN to state=OFFLINE, location=null, table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3 to restore RIT 2024-11-21T10:13:28,937 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN to state=OFFLINE, location=null, table=test, region=9226917edc2a4eac226de198b54fce9c to restore RIT 2024-11-21T10:13:28,937 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN to state=OFFLINE, location=null, table=test, region=ab23bbb87a53468249d295336f14f16f to restore RIT 2024-11-21T10:13:28,968 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184038968 2024-11-21T10:13:28,968 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:28,968 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:28,970 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:28,976 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:28,976 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:28,976 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:28,976 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,36083,1732184001229 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:28,980 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,981 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,982 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,983 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,984 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,985 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,986 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,987 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:28,988 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:28,988 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:28,988 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184068988 2024-11-21T10:13:28,988 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184128988 2024-11-21T10:13:28,988 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 19 msec 2024-11-21T10:13:28,988 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:28,988 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:28,988 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:28,988 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:28,989 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:28,989 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:28,989 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:28,991 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:28,992 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-21T10:13:29,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,001 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,002 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,002 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,004 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,008 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,009 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,010 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,010 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,010 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,011 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,011 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,012 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,012 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,012 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,013 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,013 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,013 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,013 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,013 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,013 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,013 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,014 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure fb8bd192054304ea3eed2270c166a773, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,014 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 21 msec 2024-11-21T10:13:29,022 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,027 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,028 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,029 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,040 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,041 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,041 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:29,041 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:29,042 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN to state=OFFLINE, location=null, table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3 to restore RIT 2024-11-21T10:13:29,042 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN to state=OFFLINE, location=null, table=test, region=9226917edc2a4eac226de198b54fce9c to restore RIT 2024-11-21T10:13:29,042 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN to state=OFFLINE, location=null, table=test, region=ab23bbb87a53468249d295336f14f16f to restore RIT 2024-11-21T10:13:29,064 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039064 2024-11-21T10:13:29,064 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,065 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,067 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,072 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,072 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,072 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,073 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,36083,1732184001229 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,075 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,076 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,077 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,078 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,079 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,080 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,081 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,081 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,081 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069081 2024-11-21T10:13:29,081 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129081 2024-11-21T10:13:29,081 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 16 msec 2024-11-21T10:13:29,081 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,081 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,081 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,082 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,082 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,082 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,082 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,082 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,084 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-21T10:13:29,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,090 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,098 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,101 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,102 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,103 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,103 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,103 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,103 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,103 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,103 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,103 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,103 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,103 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,103 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,103 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,104 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,104 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,104 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,104 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,104 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,104 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,104 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,104 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,104 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,104 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,104 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 32979bec7fcef8d630309c08ee22a6e6, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,104 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 20 msec 2024-11-21T10:13:29,105 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-21T10:13:29,113 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,114 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,114 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,125 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,126 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,127 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:29,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:29,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN to state=OFFLINE, location=null, table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3 to restore RIT 2024-11-21T10:13:29,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN to state=OFFLINE, location=null, table=test, region=9226917edc2a4eac226de198b54fce9c to restore RIT 2024-11-21T10:13:29,128 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN to state=OFFLINE, location=null, table=test, region=ab23bbb87a53468249d295336f14f16f to restore RIT 2024-11-21T10:13:29,129 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,160 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039160 2024-11-21T10:13:29,160 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,160 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,162 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,167 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,167 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,167 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,175 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,175 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069175 2024-11-21T10:13:29,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129175 2024-11-21T10:13:29,175 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 14 msec 2024-11-21T10:13:29,175 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,175 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,175 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,175 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,176 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,176 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,176 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,176 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,177 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,181 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,182 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,183 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,186 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,187 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,187 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,188 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,188 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,188 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,188 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,188 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,188 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,188 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,188 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,188 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,188 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,188 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,188 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,188 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,188 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,188 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure f74cd0d96e89c26d17d931ba8b44a79d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,188 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-21T10:13:29,190 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,190 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,190 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,193 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-21T10:13:29,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,198 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,198 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,199 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,199 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,199 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,199 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:29,199 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:29,199 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN to state=OFFLINE, location=null, table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3 to restore RIT 2024-11-21T10:13:29,200 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN to state=OFFLINE, location=null, table=test, region=9226917edc2a4eac226de198b54fce9c to restore RIT 2024-11-21T10:13:29,200 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN to state=OFFLINE, location=null, table=test, region=ab23bbb87a53468249d295336f14f16f to restore RIT 2024-11-21T10:13:29,200 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039200 2024-11-21T10:13:29,200 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,200 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,202 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,205 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,205 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,205 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,208 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,209 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,210 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,211 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,212 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,212 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,212 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069212 2024-11-21T10:13:29,213 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129213 2024-11-21T10:13:29,213 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 12 msec 2024-11-21T10:13:29,213 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,213 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,213 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,213 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,213 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,213 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,213 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,214 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,214 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,217 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,218 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,220 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,221 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,221 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,221 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,221 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,221 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,221 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,221 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,221 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,221 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,222 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,222 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 04a9f41b735b6d4d1061d11055315003, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,222 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-21T10:13:29,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,223 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,223 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,234 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,236 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,237 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,237 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:29,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:29,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN to state=OFFLINE, location=null, table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3 to restore RIT 2024-11-21T10:13:29,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN to state=OFFLINE, location=null, table=test, region=9226917edc2a4eac226de198b54fce9c to restore RIT 2024-11-21T10:13:29,238 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN to state=OFFLINE, location=null, table=test, region=ab23bbb87a53468249d295336f14f16f to restore RIT 2024-11-21T10:13:29,238 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039238 2024-11-21T10:13:29,238 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,239 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,240 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,245 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,245 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,246 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,246 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,246 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,247 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,248 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,249 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,249 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,249 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069249 2024-11-21T10:13:29,250 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129249 2024-11-21T10:13:29,250 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-21T10:13:29,250 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,250 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,250 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,250 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,250 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,250 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,250 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,250 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,251 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,255 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,256 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,256 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,257 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,260 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,260 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,260 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,260 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,260 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,260 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,260 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,260 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,261 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9226917edc2a4eac226de198b54fce9c, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,261 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-21T10:13:29,262 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,262 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,262 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,272 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:29,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:29,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN to state=OFFLINE, location=null, table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3 to restore RIT 2024-11-21T10:13:29,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN to state=OFFLINE, location=null, table=test, region=9226917edc2a4eac226de198b54fce9c to restore RIT 2024-11-21T10:13:29,273 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN to state=OFFLINE, location=null, table=test, region=ab23bbb87a53468249d295336f14f16f to restore RIT 2024-11-21T10:13:29,273 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039273 2024-11-21T10:13:29,274 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,274 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,275 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,277 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,277 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,277 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,282 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,283 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,284 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,285 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,285 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,285 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069285 2024-11-21T10:13:29,285 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129285 2024-11-21T10:13:29,285 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-21T10:13:29,285 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,285 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,285 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,286 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,286 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,286 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,286 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,286 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,287 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,291 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,292 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,292 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,292 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,292 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,292 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,292 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,293 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,296 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,297 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,297 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,297 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,297 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,297 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,298 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,298 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,298 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,298 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,298 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,298 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,298 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,298 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=68, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,298 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 10 msec 2024-11-21T10:13:29,299 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,299 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,299 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,310 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:29,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:29,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN to state=OFFLINE, location=null, table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3 to restore RIT 2024-11-21T10:13:29,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN to state=OFFLINE, location=null, table=test, region=9226917edc2a4eac226de198b54fce9c to restore RIT 2024-11-21T10:13:29,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN to state=OFFLINE, location=null, table=test, region=ab23bbb87a53468249d295336f14f16f to restore RIT 2024-11-21T10:13:29,311 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039311 2024-11-21T10:13:29,311 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,311 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,313 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,316 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,316 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,317 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,318 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,319 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,320 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,321 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,321 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069321 2024-11-21T10:13:29,321 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129321 2024-11-21T10:13:29,321 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-21T10:13:29,322 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,322 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,322 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,322 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,322 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,322 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,322 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,322 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,323 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,326 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,326 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,327 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,328 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,330 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,331 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,331 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,331 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,331 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,331 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,332 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,332 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,332 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,332 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,332 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN 2024-11-21T10:13:29,332 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-21T10:13:29,333 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,333 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,333 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,338 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,339 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,345 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,345 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,346 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:29,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:29,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN to state=OFFLINE, location=null, table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3 to restore RIT 2024-11-21T10:13:29,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN to state=OFFLINE, location=null, table=test, region=9226917edc2a4eac226de198b54fce9c to restore RIT 2024-11-21T10:13:29,347 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=ab23bbb87a53468249d295336f14f16f, ASSIGN to state=OFFLINE, location=null, table=test, region=ab23bbb87a53468249d295336f14f16f to restore RIT 2024-11-21T10:13:29,348 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039348 2024-11-21T10:13:29,348 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,348 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,349 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,350 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,350 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,350 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,350 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,354 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,355 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,356 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,357 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,358 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,358 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,358 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069358 2024-11-21T10:13:29,358 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129358 2024-11-21T10:13:29,358 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-21T10:13:29,358 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,358 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,358 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,359 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,359 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,359 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,359 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,359 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,360 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,362 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,363 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,363 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,365 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,366 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,366 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,366 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,366 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,366 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,366 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,367 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,367 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,367 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,367 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN 2024-11-21T10:13:29,367 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-21T10:13:29,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,368 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,368 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,377 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,377 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:29,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:29,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN to state=OFFLINE, location=null, table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3 to restore RIT 2024-11-21T10:13:29,378 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=9226917edc2a4eac226de198b54fce9c, ASSIGN to state=OFFLINE, location=null, table=test, region=9226917edc2a4eac226de198b54fce9c to restore RIT 2024-11-21T10:13:29,379 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039379 2024-11-21T10:13:29,379 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,379 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,381 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,382 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,382 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,382 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,382 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,385 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,386 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,387 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,388 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,389 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,390 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,390 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069390 2024-11-21T10:13:29,390 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129390 2024-11-21T10:13:29,390 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 11 msec 2024-11-21T10:13:29,390 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,390 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,390 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,390 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,391 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,391 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,391 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,391 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,392 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,393 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,394 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,395 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,396 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,396 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,396 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,396 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,396 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,396 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,397 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,397 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN 2024-11-21T10:13:29,397 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T10:13:29,397 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,398 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,398 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,409 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,410 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,410 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,410 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,410 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:29,410 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:29,411 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3, ASSIGN to state=OFFLINE, location=null, table=test, region=60a0ac92d2da0863bf4f5f1aac0c8da3 to restore RIT 2024-11-21T10:13:29,411 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039411 2024-11-21T10:13:29,411 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,411 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,412 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,414 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,414 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,414 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,414 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,419 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,420 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,421 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,421 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,421 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069421 2024-11-21T10:13:29,421 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129421 2024-11-21T10:13:29,421 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 10 msec 2024-11-21T10:13:29,422 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,422 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,422 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,422 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,422 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,422 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,423 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,423 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,425 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,426 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,426 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,427 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,428 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,428 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,428 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,428 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,428 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,428 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,428 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN 2024-11-21T10:13:29,428 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T10:13:29,429 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,429 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,429 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,440 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,440 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,441 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,441 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,441 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,441 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:29,442 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d, ASSIGN to state=OFFLINE, location=null, table=test, region=7b19ab03ffba010ed2e3bb6ecb72e84d to restore RIT 2024-11-21T10:13:29,442 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039442 2024-11-21T10:13:29,442 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,442 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,444 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,445 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,445 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,445 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,445 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,449 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,450 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,451 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,452 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,452 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069452 2024-11-21T10:13:29,452 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129452 2024-11-21T10:13:29,452 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T10:13:29,452 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,452 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,452 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,452 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,452 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,452 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,452 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,453 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,453 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,455 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,456 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,457 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,457 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,458 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN 2024-11-21T10:13:29,458 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T10:13:29,458 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,459 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,459 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,472 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,472 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,473 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,473 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,473 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=04a9f41b735b6d4d1061d11055315003, ASSIGN to state=OFFLINE, location=null, table=test, region=04a9f41b735b6d4d1061d11055315003 to restore RIT 2024-11-21T10:13:29,473 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039473 2024-11-21T10:13:29,473 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,473 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,475 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,476 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,476 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,477 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,477 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,479 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,480 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,481 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,482 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,483 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,483 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,483 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069483 2024-11-21T10:13:29,483 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129483 2024-11-21T10:13:29,483 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T10:13:29,483 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,483 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,483 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,483 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,483 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,483 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,484 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,484 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,485 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,486 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,486 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,487 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,487 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,488 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,489 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,489 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,490 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,490 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,490 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,490 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,490 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,490 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,490 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN 2024-11-21T10:13:29,490 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T10:13:29,492 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,492 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,492 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,503 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,503 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,503 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,504 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,515 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f74cd0d96e89c26d17d931ba8b44a79d, ASSIGN to state=OFFLINE, location=null, table=test, region=f74cd0d96e89c26d17d931ba8b44a79d to restore RIT 2024-11-21T10:13:29,517 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039517 2024-11-21T10:13:29,517 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,517 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,518 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,520 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,520 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,521 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,521 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,526 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,526 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,526 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069526 2024-11-21T10:13:29,526 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129526 2024-11-21T10:13:29,526 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T10:13:29,526 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,526 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,526 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,526 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,532 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,532 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,532 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,532 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,533 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,535 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,536 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,537 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,538 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,538 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN 2024-11-21T10:13:29,538 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-21T10:13:29,541 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,541 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,541 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,551 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,552 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,584 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,584 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,584 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=32979bec7fcef8d630309c08ee22a6e6, ASSIGN to state=OFFLINE, location=null, table=test, region=32979bec7fcef8d630309c08ee22a6e6 to restore RIT 2024-11-21T10:13:29,585 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039585 2024-11-21T10:13:29,585 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,585 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,586 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,587 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,587 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,587 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,588 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,590 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,591 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,593 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,593 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069593 2024-11-21T10:13:29,593 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129593 2024-11-21T10:13:29,593 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 7 msec 2024-11-21T10:13:29,593 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,593 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,593 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,593 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,593 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,593 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,593 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,594 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,594 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,596 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,596 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,597 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,598 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,598 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN 2024-11-21T10:13:29,598 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T10:13:29,599 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,599 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,599 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,619 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,619 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,619 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,619 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,620 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=fb8bd192054304ea3eed2270c166a773, ASSIGN to state=OFFLINE, location=null, table=test, region=fb8bd192054304ea3eed2270c166a773 to restore RIT 2024-11-21T10:13:29,620 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039620 2024-11-21T10:13:29,621 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,621 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,623 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,624 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,624 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,624 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,626 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,627 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,628 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,629 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,629 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,629 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069629 2024-11-21T10:13:29,630 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129629 2024-11-21T10:13:29,630 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T10:13:29,630 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,630 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,630 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,630 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,630 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,630 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,630 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,631 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,641 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 9 msec 2024-11-21T10:13:29,642 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,642 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,642 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,643 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,643 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,643 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,643 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,643 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,644 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,645 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,645 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-21T10:13:29,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,646 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,647 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,657 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,657 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,658 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039658 2024-11-21T10:13:29,658 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,658 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,659 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,661 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,661 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,661 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,661 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,665 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,665 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069665 2024-11-21T10:13:29,665 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129665 2024-11-21T10:13:29,665 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-21T10:13:29,665 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,665 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,665 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,665 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,665 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,666 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,666 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,666 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,666 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,668 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,669 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.1732184003140, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,669 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T10:13:29,670 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,670 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,670 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,683 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039683 2024-11-21T10:13:29,683 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,683 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,684 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,687 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,687 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,689 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,690 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,693 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,693 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069693 2024-11-21T10:13:29,693 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129693 2024-11-21T10:13:29,693 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T10:13:29,693 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,693 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,693 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,693 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,694 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,694 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,694 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,694 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,695 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,697 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-21T10:13:29,697 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,698 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,698 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,699 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,699 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,699 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,699 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,699 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T10:13:29,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,700 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,700 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,710 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,710 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,711 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039711 2024-11-21T10:13:29,711 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,711 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,712 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,714 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,714 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,714 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,714 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,716 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,717 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,718 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,719 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,720 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,720 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069720 2024-11-21T10:13:29,720 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129720 2024-11-21T10:13:29,720 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 9 msec 2024-11-21T10:13:29,720 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,720 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,720 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,720 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,720 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,720 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.CallRunner(138): callId: 85 service: RegionServerStatusService methodName: ReportRegionStateTransition size: 126 connection: 172.17.0.2:37803 deadline: 1732184019716, exception=org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false 2024-11-21T10:13:29,720 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,720 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,721 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,722 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,722 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2367): Failed report transition server { host_name: "2651e482101a" port: 36761 start_code: 1732184001491 } transition { transition_code: OPENED region_info { region_id: 1732184003970 table_name { namespace: "default" qualifier: "test" } start_key: "ooo" end_key: "ppp" offline: false split: false replica_id: 0 } open_seq_num: 2 proc_id: 79 initiating_master_active_time: 1732184001663 }; retry (#1) after 2001ms delay (Master is coming online...). org.apache.hadoop.hbase.PleaseHoldException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.makeIOExceptionOfException(ProtobufUtil.java:403) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil.getRemoteException(ProtobufUtil.java:381) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2358) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.PleaseHoldException: Master not fully online; hbase:meta=false, metaLoaded=false at org.apache.hadoop.hbase.master.assignment.AssignmentManager.checkMetaLoaded(AssignmentManager.java:1924) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.updateRegionTransition(AssignmentManager.java:1288) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1214) at org.apache.hadoop.hbase.master.assignment.AssignmentManager.reportRegionStateTransition(AssignmentManager.java:1264) at org.apache.hadoop.hbase.master.MasterRpcServices.reportRegionStateTransition(MasterRpcServices.java:1867) at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16722) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] ... 1 more 2024-11-21T10:13:29,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,724 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-21T10:13:29,724 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,725 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-21T10:13:29,725 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,725 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,725 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,726 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,726 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-21T10:13:29,726 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,727 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,727 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,735 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,735 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,735 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,736 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039736 2024-11-21T10:13:29,736 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,736 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,738 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,739 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,739 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,739 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,739 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,740 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,741 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,742 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,743 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,743 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,743 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069743 2024-11-21T10:13:29,743 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129743 2024-11-21T10:13:29,743 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-21T10:13:29,743 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,743 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,743 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,743 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,743 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,743 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,743 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,743 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,744 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,745 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-21T10:13:29,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,746 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,747 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,747 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,747 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,747 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T10:13:29,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,747 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,748 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,756 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,757 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039757 2024-11-21T10:13:29,757 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,757 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,758 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,759 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,759 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,759 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,761 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,762 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,763 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,763 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,763 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069763 2024-11-21T10:13:29,763 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129763 2024-11-21T10:13:29,763 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-21T10:13:29,763 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,763 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,763 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,763 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,763 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,763 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,763 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,764 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,764 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,766 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-21T10:13:29,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,766 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,767 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 2024-11-21T10:13:29,767 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 2651e482101a%2C33515%2C1732184001551.meta.1732184003560.meta, worker=2651e482101a,36761,1732184001491 didn't hold the lock before restarting, skip acquiring lock. 2024-11-21T10:13:29,767 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T10:13:29,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,768 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,768 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,777 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,778 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039778 2024-11-21T10:13:29,778 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,778 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,782 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-21T10:13:29,782 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,782 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,782 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,783 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,784 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,785 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,785 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069785 2024-11-21T10:13:29,785 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129785 2024-11-21T10:13:29,785 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 6 msec 2024-11-21T10:13:29,785 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-21T10:13:29,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,785 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:29,785 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:29,785 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-21T10:13:29,785 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-21T10:13:29,785 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-21T10:13:29,786 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-21T10:13:29,786 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-21T10:13:29,788 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-21T10:13:29,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-21T10:13:29,788 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-21T10:13:29,789 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true 2024-11-21T10:13:29,789 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-21T10:13:29,789 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-21T10:13:29,789 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=61 2024-11-21T10:13:29,790 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=35251 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,790 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=2651e482101a,36761,1732184001491, table=hbase:meta, region=1588230740 2024-11-21T10:13:29,790 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2651e482101a,36761,1732184001491, state=OPEN 2024-11-21T10:13:29,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-21T10:13:29,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,798 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-21T10:13:29,799 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732184039799 2024-11-21T10:13:29,799 DEBUG [Time-limited test {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-21T10:13:29,799 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-21T10:13:29,800 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-21T10:13:29,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=72, ppid=68, state=RUNNABLE, hasLock=true; OpenRegionProcedure 7b19ab03ffba010ed2e3bb6ecb72e84d, server=2651e482101a,36083,1732184001229:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 2651e482101a,33515,1732184001551, splitWal=true, meta=true exec-time=3.3280 sec 2024-11-21T10:13:29,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ed68f59ece209cbf633920e19d38f8f5, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,804 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88fcaf42f6cd2d2a83b7023a1344f960, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=60a0ac92d2da0863bf4f5f1aac0c8da3, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7b19ab03ffba010ed2e3bb6ecb72e84d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0fbe63f3c83c91920dd0c08974ab5ea8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8b2fc4f9ae6f60de0b3c78d355788c60, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1cda3137e35dc414393695b95490205a, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=90d58495640c0c466573821e63530c34, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=04a9f41b735b6d4d1061d11055315003, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=34a8113c45096a8a44b690d6f96ba3b3, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,805 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=52a49023e46e4cdb48a9866360492959, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f74cd0d96e89c26d17d931ba8b44a79d, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b207ca8401450ab08698a420b743f234, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b5794ef6bcf8b6f343b167ad897a2aa0, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ab23bbb87a53468249d295336f14f16f, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=d0e3296740e89bbfb20e449acd71bdd4, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b1d292547267d9a1644f4ca59eacbdfa, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=32979bec7fcef8d630309c08ee22a6e6, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9226917edc2a4eac226de198b54fce9c, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e90dd76cb8307eb049be31deaec171dd, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ba34ea0b7addcd538aef55c8823e35dc, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,806 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fb8bd192054304ea3eed2270c166a773, regionState=OPENING, lastHost=2651e482101a,33515,1732184001551, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=51fb43bd943dd75b745159a3a38956c8, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=12bdde3e5981435702484f9c042b20c9, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=019a4c4ad79900668e68f2029ed4ec91, regionState=OPEN, lastHost=2651e482101a,36761,1732184001491, regionLocation=2651e482101a,36761,1732184001491, openSeqNum=2 2024-11-21T10:13:29,807 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=460b55b3159e046531bc058dc9b7c499, regionState=OPEN, lastHost=2651e482101a,36083,1732184001229, regionLocation=2651e482101a,36083,1732184001229, openSeqNum=2 2024-11-21T10:13:29,807 INFO [Time-limited test {}] assignment.AssignmentManager(1776): Number of RegionServers=2 2024-11-21T10:13:29,807 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732184069807 2024-11-21T10:13:29,807 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732184129807 2024-11-21T10:13:29,807 INFO [Time-limited test {}] assignment.AssignmentManager(1783): Joined the cluster in 8 msec 2024-11-21T10:13:29,823 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=351 (was 385), OpenFileDescriptor=628 (was 651), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=368 (was 368), ProcessCount=11 (was 11), AvailableMemoryMB=2593 (was 2717) 2024-11-21T10:13:29,824 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-21T10:13:29,824 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T10:13:29,824 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T10:13:29,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T10:13:29,824 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T10:13:29,825 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-21T10:13:29,825 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-21T10:13:29,825 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2048927636, stopped=false 2024-11-21T10:13:29,825 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2651e482101a,35251,1732184000477 2024-11-21T10:13:29,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T10:13:29,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T10:13:29,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-21T10:13:29,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:29,836 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:29,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:29,836 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:29,836 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-21T10:13:29,836 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T10:13:29,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T10:13:29,836 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T10:13:29,836 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T10:13:29,837 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-21T10:13:29,837 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2651e482101a,36083,1732184001229' ***** 2024-11-21T10:13:29,837 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T10:13:29,837 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2651e482101a,36761,1732184001491' ***** 2024-11-21T10:13:29,837 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-21T10:13:29,837 INFO [RS:1;2651e482101a:36761 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T10:13:29,837 INFO [RS:0;2651e482101a:36083 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-21T10:13:29,837 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T10:13:29,837 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-21T10:13:29,837 INFO [RS:1;2651e482101a:36761 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T10:13:29,837 INFO [RS:0;2651e482101a:36083 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-21T10:13:29,837 INFO [RS:0;2651e482101a:36083 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T10:13:29,837 INFO [RS:1;2651e482101a:36761 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-21T10:13:29,838 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(3091): Received CLOSE for d0e3296740e89bbfb20e449acd71bdd4 2024-11-21T10:13:29,838 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:29,838 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(3091): Received CLOSE for 1cda3137e35dc414393695b95490205a 2024-11-21T10:13:29,838 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(3091): Received CLOSE for 52a49023e46e4cdb48a9866360492959 2024-11-21T10:13:29,838 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(3091): Received CLOSE for 8b2fc4f9ae6f60de0b3c78d355788c60 2024-11-21T10:13:29,838 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(3091): Received CLOSE for 88fcaf42f6cd2d2a83b7023a1344f960 2024-11-21T10:13:29,838 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(3091): Received CLOSE for b207ca8401450ab08698a420b743f234 2024-11-21T10:13:29,838 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(3091): Received CLOSE for e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:29,838 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(3091): Received CLOSE for 019a4c4ad79900668e68f2029ed4ec91 2024-11-21T10:13:29,839 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(3091): Received CLOSE for 12bdde3e5981435702484f9c042b20c9 2024-11-21T10:13:29,839 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(959): stopping server 2651e482101a,36761,1732184001491 2024-11-21T10:13:29,839 INFO [RS:1;2651e482101a:36761 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T10:13:29,839 INFO [RS:1;2651e482101a:36761 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;2651e482101a:36761. 2024-11-21T10:13:29,839 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing d0e3296740e89bbfb20e449acd71bdd4, disabling compactions & flushes 2024-11-21T10:13:29,839 DEBUG [RS:1;2651e482101a:36761 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T10:13:29,839 DEBUG [RS:1;2651e482101a:36761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T10:13:29,839 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:29,839 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:29,839 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. after waiting 0 ms 2024-11-21T10:13:29,839 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:29,839 INFO [RS:1;2651e482101a:36761 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T10:13:29,839 INFO [RS:1;2651e482101a:36761 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T10:13:29,839 INFO [RS:1;2651e482101a:36761 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T10:13:29,839 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for 460b55b3159e046531bc058dc9b7c499 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for ba34ea0b7addcd538aef55c8823e35dc 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for 32979bec7fcef8d630309c08ee22a6e6 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for ed68f59ece209cbf633920e19d38f8f5 2024-11-21T10:13:29,840 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fb8bd192054304ea3eed2270c166a773, disabling compactions & flushes 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for 7b19ab03ffba010ed2e3bb6ecb72e84d 2024-11-21T10:13:29,840 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for b5794ef6bcf8b6f343b167ad897a2aa0 2024-11-21T10:13:29,840 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for f74cd0d96e89c26d17d931ba8b44a79d 2024-11-21T10:13:29,840 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. after waiting 0 ms 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for 0fbe63f3c83c91920dd0c08974ab5ea8 2024-11-21T10:13:29,840 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for 90d58495640c0c466573821e63530c34 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for b1d292547267d9a1644f4ca59eacbdfa 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for 34a8113c45096a8a44b690d6f96ba3b3 2024-11-21T10:13:29,840 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(3091): Received CLOSE for 51fb43bd943dd75b745159a3a38956c8 2024-11-21T10:13:29,841 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(959): stopping server 2651e482101a,36083,1732184001229 2024-11-21T10:13:29,841 INFO [RS:0;2651e482101a:36083 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T10:13:29,841 INFO [RS:0;2651e482101a:36083 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2651e482101a:36083. 2024-11-21T10:13:29,841 DEBUG [RS:0;2651e482101a:36083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-21T10:13:29,841 DEBUG [RS:0;2651e482101a:36083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T10:13:29,841 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-21T10:13:29,842 DEBUG [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(1351): Waiting on 0fbe63f3c83c91920dd0c08974ab5ea8, 32979bec7fcef8d630309c08ee22a6e6, 34a8113c45096a8a44b690d6f96ba3b3, 460b55b3159e046531bc058dc9b7c499, 51fb43bd943dd75b745159a3a38956c8, 7b19ab03ffba010ed2e3bb6ecb72e84d, 90d58495640c0c466573821e63530c34, b1d292547267d9a1644f4ca59eacbdfa, b5794ef6bcf8b6f343b167ad897a2aa0, ba34ea0b7addcd538aef55c8823e35dc, ed68f59ece209cbf633920e19d38f8f5, f74cd0d96e89c26d17d931ba8b44a79d, fb8bd192054304ea3eed2270c166a773 2024-11-21T10:13:29,842 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(1321): Waiting on 10 regions to close 2024-11-21T10:13:29,842 DEBUG [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(1351): Waiting on 019a4c4ad79900668e68f2029ed4ec91, 12bdde3e5981435702484f9c042b20c9, 1588230740, 1cda3137e35dc414393695b95490205a, 52a49023e46e4cdb48a9866360492959, 88fcaf42f6cd2d2a83b7023a1344f960, 8b2fc4f9ae6f60de0b3c78d355788c60, ab23bbb87a53468249d295336f14f16f, b207ca8401450ab08698a420b743f234, d0e3296740e89bbfb20e449acd71bdd4, e90dd76cb8307eb049be31deaec171dd 2024-11-21T10:13:29,842 DEBUG [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-21T10:13:29,842 INFO [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-21T10:13:29,842 DEBUG [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-21T10:13:29,842 DEBUG [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-21T10:13:29,842 DEBUG [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-21T10:13:29,843 INFO [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-21T10:13:29,846 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/d0e3296740e89bbfb20e449acd71bdd4/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,849 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:29,849 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for d0e3296740e89bbfb20e449acd71bdd4: Waiting for close lock at 1732184009838Running coprocessor pre-close hooks at 1732184009839 (+1 ms)Disabling compacts and flushes for region at 1732184009839Disabling writes for close at 1732184009839Writing region close event to WAL at 1732184009840 (+1 ms)Running coprocessor post-close hooks at 1732184009847 (+7 ms)Closed at 1732184009849 (+2 ms) 2024-11-21T10:13:29,850 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732184003970.d0e3296740e89bbfb20e449acd71bdd4. 2024-11-21T10:13:29,850 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 1cda3137e35dc414393695b95490205a, disabling compactions & flushes 2024-11-21T10:13:29,850 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:29,850 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:29,850 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/fb8bd192054304ea3eed2270c166a773/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,850 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. after waiting 0 ms 2024-11-21T10:13:29,850 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:29,851 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:29,851 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fb8bd192054304ea3eed2270c166a773: Waiting for close lock at 1732184009840Running coprocessor pre-close hooks at 1732184009840Disabling compacts and flushes for region at 1732184009840Disabling writes for close at 1732184009840Writing region close event to WAL at 1732184009846 (+6 ms)Running coprocessor post-close hooks at 1732184009851 (+5 ms)Closed at 1732184009851 2024-11-21T10:13:29,851 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732184003970.fb8bd192054304ea3eed2270c166a773. 2024-11-21T10:13:29,851 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 460b55b3159e046531bc058dc9b7c499, disabling compactions & flushes 2024-11-21T10:13:29,851 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:29,851 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:29,851 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. after waiting 0 ms 2024-11-21T10:13:29,852 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:29,856 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/460b55b3159e046531bc058dc9b7c499/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,856 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/1cda3137e35dc414393695b95490205a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,857 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:29,857 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 1cda3137e35dc414393695b95490205a: Waiting for close lock at 1732184009850Running coprocessor pre-close hooks at 1732184009850Disabling compacts and flushes for region at 1732184009850Disabling writes for close at 1732184009850Writing region close event to WAL at 1732184009851 (+1 ms)Running coprocessor post-close hooks at 1732184009857 (+6 ms)Closed at 1732184009857 2024-11-21T10:13:29,857 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:29,857 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 460b55b3159e046531bc058dc9b7c499: Waiting for close lock at 1732184009851Running coprocessor pre-close hooks at 1732184009851Disabling compacts and flushes for region at 1732184009851Disabling writes for close at 1732184009851Writing region close event to WAL at 1732184009852 (+1 ms)Running coprocessor post-close hooks at 1732184009857 (+5 ms)Closed at 1732184009857 2024-11-21T10:13:29,857 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732184003970.1cda3137e35dc414393695b95490205a. 2024-11-21T10:13:29,857 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732184003970.460b55b3159e046531bc058dc9b7c499. 2024-11-21T10:13:29,857 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 52a49023e46e4cdb48a9866360492959, disabling compactions & flushes 2024-11-21T10:13:29,857 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ba34ea0b7addcd538aef55c8823e35dc, disabling compactions & flushes 2024-11-21T10:13:29,857 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:29,858 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:29,858 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:29,858 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:29,858 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. after waiting 0 ms 2024-11-21T10:13:29,858 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. after waiting 0 ms 2024-11-21T10:13:29,858 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:29,858 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:29,862 DEBUG [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/info/cda3a82551e84df5930560e169d98260 is 138, key is test,ccc,1732184003970.60a0ac92d2da0863bf4f5f1aac0c8da3./info:regioninfo/1732184008365/Put/seqid=0 2024-11-21T10:13:29,873 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ba34ea0b7addcd538aef55c8823e35dc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,874 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:29,874 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ba34ea0b7addcd538aef55c8823e35dc: Waiting for close lock at 1732184009857Running coprocessor pre-close hooks at 1732184009857Disabling compacts and flushes for region at 1732184009857Disabling writes for close at 1732184009858 (+1 ms)Writing region close event to WAL at 1732184009862 (+4 ms)Running coprocessor post-close hooks at 1732184009874 (+12 ms)Closed at 1732184009874 2024-11-21T10:13:29,874 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732184003970.ba34ea0b7addcd538aef55c8823e35dc. 2024-11-21T10:13:29,874 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 32979bec7fcef8d630309c08ee22a6e6, disabling compactions & flushes 2024-11-21T10:13:29,875 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:29,875 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:29,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741869_1045 (size=8195) 2024-11-21T10:13:29,875 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. after waiting 0 ms 2024-11-21T10:13:29,875 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:29,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741869_1045 (size=8195) 2024-11-21T10:13:29,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741869_1045 (size=8195) 2024-11-21T10:13:29,876 INFO [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/info/cda3a82551e84df5930560e169d98260 2024-11-21T10:13:29,877 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/52a49023e46e4cdb48a9866360492959/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,878 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:29,878 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 52a49023e46e4cdb48a9866360492959: Waiting for close lock at 1732184009857Running coprocessor pre-close hooks at 1732184009857Disabling compacts and flushes for region at 1732184009857Disabling writes for close at 1732184009858 (+1 ms)Writing region close event to WAL at 1732184009862 (+4 ms)Running coprocessor post-close hooks at 1732184009878 (+16 ms)Closed at 1732184009878 2024-11-21T10:13:29,878 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732184003970.52a49023e46e4cdb48a9866360492959. 2024-11-21T10:13:29,878 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8b2fc4f9ae6f60de0b3c78d355788c60, disabling compactions & flushes 2024-11-21T10:13:29,878 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:29,878 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:29,878 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. after waiting 0 ms 2024-11-21T10:13:29,878 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:29,881 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/32979bec7fcef8d630309c08ee22a6e6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,883 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:29,883 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 32979bec7fcef8d630309c08ee22a6e6: Waiting for close lock at 1732184009874Running coprocessor pre-close hooks at 1732184009874Disabling compacts and flushes for region at 1732184009874Disabling writes for close at 1732184009875 (+1 ms)Writing region close event to WAL at 1732184009876 (+1 ms)Running coprocessor post-close hooks at 1732184009883 (+7 ms)Closed at 1732184009883 2024-11-21T10:13:29,883 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732184003970.32979bec7fcef8d630309c08ee22a6e6. 2024-11-21T10:13:29,883 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ed68f59ece209cbf633920e19d38f8f5, disabling compactions & flushes 2024-11-21T10:13:29,883 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:29,883 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:29,883 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. after waiting 0 ms 2024-11-21T10:13:29,883 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:29,884 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/8b2fc4f9ae6f60de0b3c78d355788c60/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,885 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:29,885 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8b2fc4f9ae6f60de0b3c78d355788c60: Waiting for close lock at 1732184009878Running coprocessor pre-close hooks at 1732184009878Disabling compacts and flushes for region at 1732184009878Disabling writes for close at 1732184009878Writing region close event to WAL at 1732184009879 (+1 ms)Running coprocessor post-close hooks at 1732184009885 (+6 ms)Closed at 1732184009885 2024-11-21T10:13:29,885 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732184003970.8b2fc4f9ae6f60de0b3c78d355788c60. 2024-11-21T10:13:29,885 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88fcaf42f6cd2d2a83b7023a1344f960, disabling compactions & flushes 2024-11-21T10:13:29,886 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:29,886 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:29,886 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. after waiting 0 ms 2024-11-21T10:13:29,886 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:29,887 DEBUG [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/.tmp/info/cda3a82551e84df5930560e169d98260 as hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/info/cda3a82551e84df5930560e169d98260 2024-11-21T10:13:29,888 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/ed68f59ece209cbf633920e19d38f8f5/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,889 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:29,890 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ed68f59ece209cbf633920e19d38f8f5: Waiting for close lock at 1732184009883Running coprocessor pre-close hooks at 1732184009883Disabling compacts and flushes for region at 1732184009883Disabling writes for close at 1732184009883Writing region close event to WAL at 1732184009884 (+1 ms)Running coprocessor post-close hooks at 1732184009889 (+5 ms)Closed at 1732184009889 2024-11-21T10:13:29,890 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732184003970.ed68f59ece209cbf633920e19d38f8f5. 2024-11-21T10:13:29,890 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7b19ab03ffba010ed2e3bb6ecb72e84d, disabling compactions & flushes 2024-11-21T10:13:29,890 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:29,890 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:29,890 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. after waiting 0 ms 2024-11-21T10:13:29,890 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:29,891 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/88fcaf42f6cd2d2a83b7023a1344f960/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,891 INFO [regionserver/2651e482101a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T10:13:29,892 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:29,892 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88fcaf42f6cd2d2a83b7023a1344f960: Waiting for close lock at 1732184009885Running coprocessor pre-close hooks at 1732184009885Disabling compacts and flushes for region at 1732184009885Disabling writes for close at 1732184009886 (+1 ms)Writing region close event to WAL at 1732184009886Running coprocessor post-close hooks at 1732184009892 (+6 ms)Closed at 1732184009892 2024-11-21T10:13:29,893 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732184003970.88fcaf42f6cd2d2a83b7023a1344f960. 2024-11-21T10:13:29,893 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b207ca8401450ab08698a420b743f234, disabling compactions & flushes 2024-11-21T10:13:29,893 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:29,893 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:29,893 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. after waiting 0 ms 2024-11-21T10:13:29,893 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:29,896 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/7b19ab03ffba010ed2e3bb6ecb72e84d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,897 INFO [regionserver/2651e482101a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T10:13:29,898 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:29,898 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7b19ab03ffba010ed2e3bb6ecb72e84d: Waiting for close lock at 1732184009890Running coprocessor pre-close hooks at 1732184009890Disabling compacts and flushes for region at 1732184009890Disabling writes for close at 1732184009890Writing region close event to WAL at 1732184009891 (+1 ms)Running coprocessor post-close hooks at 1732184009898 (+7 ms)Closed at 1732184009898 2024-11-21T10:13:29,898 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732184003970.7b19ab03ffba010ed2e3bb6ecb72e84d. 2024-11-21T10:13:29,898 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b207ca8401450ab08698a420b743f234/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,898 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b5794ef6bcf8b6f343b167ad897a2aa0, disabling compactions & flushes 2024-11-21T10:13:29,898 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:29,898 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:29,898 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. after waiting 0 ms 2024-11-21T10:13:29,898 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:29,899 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:29,899 INFO [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/info/cda3a82551e84df5930560e169d98260, entries=24, sequenceid=71, filesize=8.0 K 2024-11-21T10:13:29,899 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b207ca8401450ab08698a420b743f234: Waiting for close lock at 1732184009893Running coprocessor pre-close hooks at 1732184009893Disabling compacts and flushes for region at 1732184009893Disabling writes for close at 1732184009893Writing region close event to WAL at 1732184009893Running coprocessor post-close hooks at 1732184009899 (+6 ms)Closed at 1732184009899 2024-11-21T10:13:29,899 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732184003970.b207ca8401450ab08698a420b743f234. 2024-11-21T10:13:29,899 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e90dd76cb8307eb049be31deaec171dd, disabling compactions & flushes 2024-11-21T10:13:29,899 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:29,900 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:29,900 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. after waiting 0 ms 2024-11-21T10:13:29,900 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:29,901 INFO [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 58ms, sequenceid=71, compaction requested=false 2024-11-21T10:13:29,904 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b5794ef6bcf8b6f343b167ad897a2aa0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,905 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/e90dd76cb8307eb049be31deaec171dd/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,905 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:29,905 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b5794ef6bcf8b6f343b167ad897a2aa0: Waiting for close lock at 1732184009898Running coprocessor pre-close hooks at 1732184009898Disabling compacts and flushes for region at 1732184009898Disabling writes for close at 1732184009898Writing region close event to WAL at 1732184009899 (+1 ms)Running coprocessor post-close hooks at 1732184009905 (+6 ms)Closed at 1732184009905 2024-11-21T10:13:29,905 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:29,905 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732184003970.b5794ef6bcf8b6f343b167ad897a2aa0. 2024-11-21T10:13:29,905 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e90dd76cb8307eb049be31deaec171dd: Waiting for close lock at 1732184009899Running coprocessor pre-close hooks at 1732184009899Disabling compacts and flushes for region at 1732184009899Disabling writes for close at 1732184009900 (+1 ms)Writing region close event to WAL at 1732184009900Running coprocessor post-close hooks at 1732184009905 (+5 ms)Closed at 1732184009905 2024-11-21T10:13:29,906 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f74cd0d96e89c26d17d931ba8b44a79d, disabling compactions & flushes 2024-11-21T10:13:29,906 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732184003970.e90dd76cb8307eb049be31deaec171dd. 2024-11-21T10:13:29,906 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:29,906 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:29,906 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. after waiting 0 ms 2024-11-21T10:13:29,906 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 019a4c4ad79900668e68f2029ed4ec91, disabling compactions & flushes 2024-11-21T10:13:29,906 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:29,906 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:29,906 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:29,906 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. after waiting 0 ms 2024-11-21T10:13:29,906 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:29,908 DEBUG [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-21T10:13:29,909 DEBUG [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-21T10:13:29,909 INFO [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-21T10:13:29,909 DEBUG [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732184009842Running coprocessor pre-close hooks at 1732184009842Disabling compacts and flushes for region at 1732184009842Disabling writes for close at 1732184009842Obtaining lock to block concurrent updates at 1732184009843 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1732184009843Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1732184009851 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732184009852 (+1 ms)Flushing 1588230740/info: creating writer at 1732184009852Flushing 1588230740/info: appending metadata at 1732184009862 (+10 ms)Flushing 1588230740/info: closing flushed file at 1732184009862Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4adf9fc4: reopening flushed file at 1732184009886 (+24 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 58ms, sequenceid=71, compaction requested=false at 1732184009901 (+15 ms)Writing region close event to WAL at 1732184009903 (+2 ms)Running coprocessor post-close hooks at 1732184009909 (+6 ms)Closed at 1732184009909 2024-11-21T10:13:29,910 DEBUG [RS_CLOSE_META-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-21T10:13:29,910 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/f74cd0d96e89c26d17d931ba8b44a79d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,910 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/019a4c4ad79900668e68f2029ed4ec91/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,911 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:29,911 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f74cd0d96e89c26d17d931ba8b44a79d: Waiting for close lock at 1732184009905Running coprocessor pre-close hooks at 1732184009905Disabling compacts and flushes for region at 1732184009905Disabling writes for close at 1732184009906 (+1 ms)Writing region close event to WAL at 1732184009906Running coprocessor post-close hooks at 1732184009911 (+5 ms)Closed at 1732184009911 2024-11-21T10:13:29,911 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:29,911 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 019a4c4ad79900668e68f2029ed4ec91: Waiting for close lock at 1732184009906Running coprocessor pre-close hooks at 1732184009906Disabling compacts and flushes for region at 1732184009906Disabling writes for close at 1732184009906Writing region close event to WAL at 1732184009907 (+1 ms)Running coprocessor post-close hooks at 1732184009911 (+4 ms)Closed at 1732184009911 2024-11-21T10:13:29,911 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732184003970.f74cd0d96e89c26d17d931ba8b44a79d. 2024-11-21T10:13:29,911 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732184003970.019a4c4ad79900668e68f2029ed4ec91. 2024-11-21T10:13:29,912 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0fbe63f3c83c91920dd0c08974ab5ea8, disabling compactions & flushes 2024-11-21T10:13:29,912 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 12bdde3e5981435702484f9c042b20c9, disabling compactions & flushes 2024-11-21T10:13:29,912 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:29,912 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:29,912 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:29,912 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:29,912 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. after waiting 0 ms 2024-11-21T10:13:29,912 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. after waiting 0 ms 2024-11-21T10:13:29,912 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:29,912 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:29,916 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/12bdde3e5981435702484f9c042b20c9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,916 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/0fbe63f3c83c91920dd0c08974ab5ea8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,917 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:29,917 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 12bdde3e5981435702484f9c042b20c9: Waiting for close lock at 1732184009912Running coprocessor pre-close hooks at 1732184009912Disabling compacts and flushes for region at 1732184009912Disabling writes for close at 1732184009912Writing region close event to WAL at 1732184009912Running coprocessor post-close hooks at 1732184009917 (+5 ms)Closed at 1732184009917 2024-11-21T10:13:29,917 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:29,917 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732184003970.12bdde3e5981435702484f9c042b20c9. 2024-11-21T10:13:29,917 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0fbe63f3c83c91920dd0c08974ab5ea8: Waiting for close lock at 1732184009911Running coprocessor pre-close hooks at 1732184009911Disabling compacts and flushes for region at 1732184009911Disabling writes for close at 1732184009912 (+1 ms)Writing region close event to WAL at 1732184009912Running coprocessor post-close hooks at 1732184009917 (+5 ms)Closed at 1732184009917 2024-11-21T10:13:29,917 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732184003970.0fbe63f3c83c91920dd0c08974ab5ea8. 2024-11-21T10:13:29,918 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 90d58495640c0c466573821e63530c34, disabling compactions & flushes 2024-11-21T10:13:29,918 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:29,918 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:29,918 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732184003970.90d58495640c0c466573821e63530c34. after waiting 0 ms 2024-11-21T10:13:29,918 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:29,923 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/90d58495640c0c466573821e63530c34/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,923 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:29,923 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 90d58495640c0c466573821e63530c34: Waiting for close lock at 1732184009917Running coprocessor pre-close hooks at 1732184009918 (+1 ms)Disabling compacts and flushes for region at 1732184009918Disabling writes for close at 1732184009918Writing region close event to WAL at 1732184009918Running coprocessor post-close hooks at 1732184009923 (+5 ms)Closed at 1732184009923 2024-11-21T10:13:29,924 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732184003970.90d58495640c0c466573821e63530c34. 2024-11-21T10:13:29,924 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b1d292547267d9a1644f4ca59eacbdfa, disabling compactions & flushes 2024-11-21T10:13:29,924 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:29,924 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:29,924 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. after waiting 0 ms 2024-11-21T10:13:29,924 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:29,929 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/b1d292547267d9a1644f4ca59eacbdfa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,930 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:29,930 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b1d292547267d9a1644f4ca59eacbdfa: Waiting for close lock at 1732184009924Running coprocessor pre-close hooks at 1732184009924Disabling compacts and flushes for region at 1732184009924Disabling writes for close at 1732184009924Writing region close event to WAL at 1732184009924Running coprocessor post-close hooks at 1732184009930 (+6 ms)Closed at 1732184009930 2024-11-21T10:13:29,930 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732184003970.b1d292547267d9a1644f4ca59eacbdfa. 2024-11-21T10:13:29,931 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 34a8113c45096a8a44b690d6f96ba3b3, disabling compactions & flushes 2024-11-21T10:13:29,931 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:29,931 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:29,931 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. after waiting 0 ms 2024-11-21T10:13:29,931 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:29,936 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/34a8113c45096a8a44b690d6f96ba3b3/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,937 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:29,937 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 34a8113c45096a8a44b690d6f96ba3b3: Waiting for close lock at 1732184009931Running coprocessor pre-close hooks at 1732184009931Disabling compacts and flushes for region at 1732184009931Disabling writes for close at 1732184009931Writing region close event to WAL at 1732184009931Running coprocessor post-close hooks at 1732184009937 (+6 ms)Closed at 1732184009937 2024-11-21T10:13:29,937 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732184003970.34a8113c45096a8a44b690d6f96ba3b3. 2024-11-21T10:13:29,937 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 51fb43bd943dd75b745159a3a38956c8, disabling compactions & flushes 2024-11-21T10:13:29,937 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:29,938 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:29,938 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. after waiting 0 ms 2024-11-21T10:13:29,938 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:29,943 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/data/default/test/51fb43bd943dd75b745159a3a38956c8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-21T10:13:29,943 INFO [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:29,943 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 51fb43bd943dd75b745159a3a38956c8: Waiting for close lock at 1732184009937Running coprocessor pre-close hooks at 1732184009937Disabling compacts and flushes for region at 1732184009937Disabling writes for close at 1732184009938 (+1 ms)Writing region close event to WAL at 1732184009938Running coprocessor post-close hooks at 1732184009943 (+5 ms)Closed at 1732184009943 2024-11-21T10:13:29,944 DEBUG [RS_CLOSE_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732184003970.51fb43bd943dd75b745159a3a38956c8. 2024-11-21T10:13:29,987 INFO [regionserver/2651e482101a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T10:13:29,987 INFO [regionserver/2651e482101a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T10:13:29,999 INFO [regionserver/2651e482101a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-21T10:13:29,999 INFO [regionserver/2651e482101a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-21T10:13:30,042 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(976): stopping server 2651e482101a,36083,1732184001229; all regions closed. 2024-11-21T10:13:30,042 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(976): stopping server 2651e482101a,36761,1732184001491; all regions closed. 2024-11-21T10:13:30,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741834_1010 (size=7064) 2024-11-21T10:13:30,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741865_1041 (size=4204) 2024-11-21T10:13:30,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741834_1010 (size=7064) 2024-11-21T10:13:30,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741865_1041 (size=4204) 2024-11-21T10:13:30,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741834_1010 (size=7064) 2024-11-21T10:13:30,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741865_1041 (size=4204) 2024-11-21T10:13:30,052 DEBUG [RS:1;2651e482101a:36761 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/oldWALs 2024-11-21T10:13:30,052 DEBUG [RS:0;2651e482101a:36083 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/oldWALs 2024-11-21T10:13:30,052 INFO [RS:1;2651e482101a:36761 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2651e482101a%2C36761%2C1732184001491.meta:.meta(num 1732184007163) 2024-11-21T10:13:30,052 INFO [RS:0;2651e482101a:36083 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2651e482101a%2C36083%2C1732184001229:(num 1732184003140) 2024-11-21T10:13:30,052 DEBUG [RS:0;2651e482101a:36083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T10:13:30,052 INFO [RS:0;2651e482101a:36083 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T10:13:30,052 INFO [RS:0;2651e482101a:36083 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T10:13:30,053 INFO [RS:0;2651e482101a:36083 {}] hbase.ChoreService(370): Chore service for: regionserver/2651e482101a:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-21T10:13:30,053 INFO [RS:0;2651e482101a:36083 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-21T10:13:30,053 INFO [regionserver/2651e482101a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T10:13:30,053 INFO [RS:0;2651e482101a:36083 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-21T10:13:30,053 INFO [RS:0;2651e482101a:36083 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-21T10:13:30,053 INFO [RS:0;2651e482101a:36083 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T10:13:30,053 INFO [RS:0;2651e482101a:36083 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36083 2024-11-21T10:13:30,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741833_1009 (size=5194) 2024-11-21T10:13:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741833_1009 (size=5194) 2024-11-21T10:13:30,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741833_1009 (size=5194) 2024-11-21T10:13:30,058 DEBUG [RS:1;2651e482101a:36761 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/oldWALs 2024-11-21T10:13:30,058 INFO [RS:1;2651e482101a:36761 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2651e482101a%2C36761%2C1732184001491:(num 1732184003140) 2024-11-21T10:13:30,058 DEBUG [RS:1;2651e482101a:36761 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-21T10:13:30,058 INFO [RS:1;2651e482101a:36761 {}] regionserver.LeaseManager(133): Closed leases 2024-11-21T10:13:30,058 INFO [RS:1;2651e482101a:36761 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T10:13:30,058 INFO [RS:1;2651e482101a:36761 {}] hbase.ChoreService(370): Chore service for: regionserver/2651e482101a:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-21T10:13:30,058 INFO [RS:1;2651e482101a:36761 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T10:13:30,058 INFO [regionserver/2651e482101a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T10:13:30,059 WARN [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegionServer.reportRegionStateTransition(HRegionServer.java:2374) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2257) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T10:13:30,059 WARN [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(168): Fatal error occurred while opening region test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f., aborting... java.io.IOException: Failed to report opened region to master: test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T10:13:30,059 ERROR [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2440): ***** ABORTING region server 2651e482101a,36761,1732184001491: Failed to open region test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. and can not recover ***** java.io.IOException: Failed to report opened region to master: test,ooo,1732184003970.ab23bbb87a53468249d295336f14f16f. at org.apache.hadoop.hbase.regionserver.HRegionServer.postOpenDeployTasks(HRegionServer.java:2261) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler.process(AssignRegionHandler.java:150) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-21T10:13:30,059 ERROR [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-21T10:13:30,059 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-21T10:13:30,060 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-21T10:13:30,060 INFO [RS:1;2651e482101a:36761 {}] executor.ExecutorService(108): Executor-9-RS_OPEN_REGION-regionserver/2651e482101a:0 had [Event #36 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #37 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler), Event #38 of type M_RS_OPEN_REGION (class org.apache.hadoop.hbase.regionserver.handler.AssignRegionHandler)] on shutdown 2024-11-21T10:13:30,060 INFO [RS:1;2651e482101a:36761 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36761 2024-11-21T10:13:30,061 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2651e482101a,36083,1732184001229 2024-11-21T10:13:30,061 INFO [RS:0;2651e482101a:36083 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T10:13:30,062 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-21T10:13:30,063 DEBUG [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-21T10:13:30,070 INFO [RS_OPEN_REGION-regionserver/2651e482101a:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 487327832 }, "NonHeapMemoryUsage": { "committed": 162988032, "init": 7667712, "max": -1, "used": 160306176 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "2651e482101a", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 2, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 38278, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 113, "ProcessCallTime_min": 0, "ProcessCallTime_max": 43, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 10, "ProcessCallTime_median": 21, "ProcessCallTime_75th_percentile": 32, "ProcessCallTime_90th_percentile": 38, "ProcessCallTime_95th_percentile": 40, "ProcessCallTime_98th_percentile": 42, "ProcessCallTime_99th_percentile": 42, "ProcessCallTime_99.9th_percentile": 42, "ProcessCallTime_TimeRangeCount_0-1": 113, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 113, "QueueCallTime_min": 0, "QueueCallTime_max": 14, "QueueCallTime_mean": 1, "QueueCallTime_25th_percentile": 3, "QueueCallTime_median": 7, "QueueCallTime_75th_percentile": 10, "QueueCallTime_90th_percentile": 12, "QueueCallTime_95th_percentile": 13, "QueueCallTime_98th_percentile": 13, "QueueCallTime_99th_percentile": 13, "QueueCallTime_99.9th_percentile": 13, "QueueCallTime_TimeRangeCount_0-1": 113, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 113, "TotalCallTime_min": 0, "TotalCallTime_max": 43, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 10, "TotalCallTime_median": 21, "TotalCallTime_75th_percentile": 32, "TotalCallTime_90th_percentile": 38, "TotalCallTime_95th_percentile": 40, "TotalCallTime_98th_percentile": 42, "TotalCallTime_99th_percentile": 42, "TotalCallTime_99.9th_percentile": 42, "TotalCallTime_TimeRangeCount_0-1": 113, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 113, "ResponseSize_min": 0, "ResponseSize_max": 22855, "ResponseSize_mean": 1414, "ResponseSize_25th_percentile": 5713, "ResponseSize_median": 11427, "ResponseSize_75th_percentile": 17141, "ResponseSize_90th_percentile": 20569, "ResponseSize_95th_percentile": 21712, "ResponseSize_98th_percentile": 22397, "ResponseSize_99th_percentile": 22626, "ResponseSize_99.9th_percentile": 22832, "ResponseSize_SizeRangeCount_0-10": 113, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 113, "RequestSize_min": 81, "RequestSize_max": 8095, "RequestSize_mean": 306, "RequestSize_25th_percentile": 2084, "RequestSize_median": 4088, "RequestSize_75th_percentile": 6091, "RequestSize_90th_percentile": 7293, "RequestSize_95th_percentile": 7694, "RequestSize_98th_percentile": 7934, "RequestSize_99th_percentile": 8014, "RequestSize_99.9th_percentile": 8086, "RequestSize_SizeRangeCount_0-10": 113, "sentBytes": 125265 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "2651e482101a", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:55007", "tag.serverName": "2651e482101a,36083,1732184001229", "tag.clusterId": "114bcabb-0504-4c22-8f5b-f31aa098ec4f", "tag.Context": "regionserver", "tag.Hostname": "2651e482101a", "regionCount": 13, "storeCount": 9, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 2304, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732184001229, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 3332, "localBytesRead": 3332, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 1.8, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 36711, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 9, "totalRowActionRequestCount": 9, "readRequestCount": 9, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 9, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 9, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "ScanSize_SizeRangeCount_0-10": 9, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-21T10:13:30,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-21T10:13:30,071 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2651e482101a,36761,1732184001491 2024-11-21T10:13:30,071 INFO [RS:1;2651e482101a:36761 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T10:13:30,082 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2651e482101a,36083,1732184001229] 2024-11-21T10:13:30,103 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2651e482101a,36083,1732184001229 already deleted, retry=false 2024-11-21T10:13:30,103 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2651e482101a,36083,1732184001229 expired; onlineServers=1 2024-11-21T10:13:30,103 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2651e482101a,36761,1732184001491] 2024-11-21T10:13:30,113 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2651e482101a,36761,1732184001491 already deleted, retry=false 2024-11-21T10:13:30,113 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2651e482101a,36761,1732184001491 expired; onlineServers=0 2024-11-21T10:13:30,114 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2651e482101a,35251,1732184000477' ***** 2024-11-21T10:13:30,114 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-21T10:13:30,114 INFO [M:0;2651e482101a:35251 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-21T10:13:30,114 INFO [M:0;2651e482101a:35251 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-21T10:13:30,114 DEBUG [M:0;2651e482101a:35251 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-21T10:13:30,114 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-21T10:13:30,114 DEBUG [M:0;2651e482101a:35251 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-21T10:13:30,114 DEBUG [master/2651e482101a:0:becomeActiveMaster-HFileCleaner.large.0-1732184002701 {}] cleaner.HFileCleaner(306): Exit Thread[master/2651e482101a:0:becomeActiveMaster-HFileCleaner.large.0-1732184002701,5,FailOnTimeoutGroup] 2024-11-21T10:13:30,114 DEBUG [master/2651e482101a:0:becomeActiveMaster-HFileCleaner.small.0-1732184002702 {}] cleaner.HFileCleaner(306): Exit Thread[master/2651e482101a:0:becomeActiveMaster-HFileCleaner.small.0-1732184002702,5,FailOnTimeoutGroup] 2024-11-21T10:13:30,114 INFO [M:0;2651e482101a:35251 {}] hbase.ChoreService(370): Chore service for: master/2651e482101a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-21T10:13:30,115 INFO [M:0;2651e482101a:35251 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-21T10:13:30,115 DEBUG [M:0;2651e482101a:35251 {}] master.HMaster(1795): Stopping service threads 2024-11-21T10:13:30,115 INFO [M:0;2651e482101a:35251 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-21T10:13:30,115 INFO [M:0;2651e482101a:35251 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-21T10:13:30,115 INFO [M:0;2651e482101a:35251 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-21T10:13:30,115 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-21T10:13:30,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-21T10:13:30,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-21T10:13:30,124 DEBUG [M:0;2651e482101a:35251 {}] zookeeper.ZKUtil(347): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-21T10:13:30,124 WARN [M:0;2651e482101a:35251 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-21T10:13:30,125 INFO [M:0;2651e482101a:35251 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/.lastflushedseqids 2024-11-21T10:13:30,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741870_1046 (size=1) 2024-11-21T10:13:30,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741870_1046 (size=1) 2024-11-21T10:13:30,138 INFO [M:0;2651e482101a:35251 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-21T10:13:30,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741870_1046 (size=1) 2024-11-21T10:13:30,138 INFO [M:0;2651e482101a:35251 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-21T10:13:30,139 DEBUG [M:0;2651e482101a:35251 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-21T10:13:30,139 INFO [M:0;2651e482101a:35251 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T10:13:30,139 DEBUG [M:0;2651e482101a:35251 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T10:13:30,139 DEBUG [M:0;2651e482101a:35251 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-21T10:13:30,139 DEBUG [M:0;2651e482101a:35251 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T10:13:30,139 INFO [M:0;2651e482101a:35251 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=248.12 KB heapSize=301.51 KB 2024-11-21T10:13:30,160 DEBUG [M:0;2651e482101a:35251 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b781bd051d64ed888f741e34f59a9dd is 82, key is hbase:meta,,1/info:regioninfo/1732184007935/Put/seqid=0 2024-11-21T10:13:30,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741871_1047 (size=6071) 2024-11-21T10:13:30,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741871_1047 (size=6071) 2024-11-21T10:13:30,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741871_1047 (size=6071) 2024-11-21T10:13:30,168 INFO [M:0;2651e482101a:35251 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b781bd051d64ed888f741e34f59a9dd 2024-11-21T10:13:30,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T10:13:30,182 INFO [RS:0;2651e482101a:36083 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T10:13:30,182 INFO [RS:0;2651e482101a:36083 {}] regionserver.HRegionServer(1031): Exiting; stopping=2651e482101a,36083,1732184001229; zookeeper connection closed. 2024-11-21T10:13:30,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36083-0x1015ce240570001, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T10:13:30,182 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@633b1f46 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@633b1f46 2024-11-21T10:13:30,192 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T10:13:30,193 INFO [RS:1;2651e482101a:36761 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T10:13:30,193 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36761-0x1015ce240570002, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T10:13:30,193 INFO [RS:1;2651e482101a:36761 {}] regionserver.HRegionServer(1031): Exiting; stopping=2651e482101a,36761,1732184001491; zookeeper connection closed. 2024-11-21T10:13:30,193 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@41b65a85 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@41b65a85 2024-11-21T10:13:30,193 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-21T10:13:30,194 DEBUG [M:0;2651e482101a:35251 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f4375cdad92548fa8db8291869806501 is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1732184009801/Put/seqid=0 2024-11-21T10:13:30,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741872_1048 (size=12802) 2024-11-21T10:13:30,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741872_1048 (size=12802) 2024-11-21T10:13:30,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741872_1048 (size=12802) 2024-11-21T10:13:30,204 INFO [M:0;2651e482101a:35251 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=246.84 KB at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f4375cdad92548fa8db8291869806501 2024-11-21T10:13:30,226 DEBUG [M:0;2651e482101a:35251 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/34dde2c898c24d8f9fdecca699c4f0fa is 69, key is 2651e482101a,36083,1732184001229/rs:state/1732184002849/Put/seqid=0 2024-11-21T10:13:30,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741873_1049 (size=5442) 2024-11-21T10:13:30,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741873_1049 (size=5442) 2024-11-21T10:13:30,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741873_1049 (size=5442) 2024-11-21T10:13:30,233 INFO [M:0;2651e482101a:35251 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/34dde2c898c24d8f9fdecca699c4f0fa 2024-11-21T10:13:30,239 INFO [M:0;2651e482101a:35251 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 34dde2c898c24d8f9fdecca699c4f0fa 2024-11-21T10:13:30,255 DEBUG [M:0;2651e482101a:35251 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/722502c92d344ea7b5a39f4fbb601f42 is 52, key is load_balancer_on/state:d/1732184006320/Put/seqid=0 2024-11-21T10:13:30,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741874_1050 (size=5058) 2024-11-21T10:13:30,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741874_1050 (size=5058) 2024-11-21T10:13:30,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741874_1050 (size=5058) 2024-11-21T10:13:30,262 INFO [M:0;2651e482101a:35251 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=569 (bloomFilter=true), to=hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/722502c92d344ea7b5a39f4fbb601f42 2024-11-21T10:13:30,268 DEBUG [M:0;2651e482101a:35251 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9b781bd051d64ed888f741e34f59a9dd as hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9b781bd051d64ed888f741e34f59a9dd 2024-11-21T10:13:30,274 INFO [M:0;2651e482101a:35251 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9b781bd051d64ed888f741e34f59a9dd, entries=14, sequenceid=569, filesize=5.9 K 2024-11-21T10:13:30,276 DEBUG [M:0;2651e482101a:35251 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f4375cdad92548fa8db8291869806501 as hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f4375cdad92548fa8db8291869806501 2024-11-21T10:13:30,282 INFO [M:0;2651e482101a:35251 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f4375cdad92548fa8db8291869806501, entries=79, sequenceid=569, filesize=12.5 K 2024-11-21T10:13:30,284 DEBUG [M:0;2651e482101a:35251 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/34dde2c898c24d8f9fdecca699c4f0fa as hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/34dde2c898c24d8f9fdecca699c4f0fa 2024-11-21T10:13:30,292 INFO [M:0;2651e482101a:35251 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 34dde2c898c24d8f9fdecca699c4f0fa 2024-11-21T10:13:30,292 INFO [M:0;2651e482101a:35251 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/34dde2c898c24d8f9fdecca699c4f0fa, entries=3, sequenceid=569, filesize=5.3 K 2024-11-21T10:13:30,293 DEBUG [M:0;2651e482101a:35251 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/722502c92d344ea7b5a39f4fbb601f42 as hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/722502c92d344ea7b5a39f4fbb601f42 2024-11-21T10:13:30,301 INFO [M:0;2651e482101a:35251 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:39887/user/jenkins/test-data/a3a0c4e2-0770-c437-eb08-748138ff47ff/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/722502c92d344ea7b5a39f4fbb601f42, entries=1, sequenceid=569, filesize=4.9 K 2024-11-21T10:13:30,303 INFO [M:0;2651e482101a:35251 {}] regionserver.HRegion(3140): Finished flush of dataSize ~248.12 KB/254073, heapSize ~301.45 KB/308680, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 164ms, sequenceid=569, compaction requested=false 2024-11-21T10:13:30,306 INFO [M:0;2651e482101a:35251 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-21T10:13:30,307 DEBUG [M:0;2651e482101a:35251 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732184010139Disabling compacts and flushes for region at 1732184010139Disabling writes for close at 1732184010139Obtaining lock to block concurrent updates at 1732184010139Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732184010139Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=254073, getHeapSize=308680, getOffHeapSize=0, getCellsCount=708 at 1732184010140 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732184010144 (+4 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732184010144Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732184010160 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732184010160Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732184010174 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732184010193 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732184010193Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732184010210 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732184010225 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732184010225Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1732184010239 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1732184010254 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1732184010254Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2226c878: reopening flushed file at 1732184010267 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f52f43b: reopening flushed file at 1732184010275 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@597da6d7: reopening flushed file at 1732184010283 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f86c84d: reopening flushed file at 1732184010292 (+9 ms)Finished flush of dataSize ~248.12 KB/254073, heapSize ~301.45 KB/308680, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 164ms, sequenceid=569, compaction requested=false at 1732184010303 (+11 ms)Writing region close event to WAL at 1732184010306 (+3 ms)Closed at 1732184010306 2024-11-21T10:13:30,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45815 is added to blk_1073741830_1006 (size=292067) 2024-11-21T10:13:30,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741830_1006 (size=292067) 2024-11-21T10:13:30,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39431 is added to blk_1073741830_1006 (size=292067) 2024-11-21T10:13:30,310 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-21T10:13:30,310 INFO [M:0;2651e482101a:35251 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-21T10:13:30,310 INFO [M:0;2651e482101a:35251 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:35251 2024-11-21T10:13:30,311 INFO [M:0;2651e482101a:35251 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-21T10:13:30,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T10:13:30,424 INFO [M:0;2651e482101a:35251 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-21T10:13:30,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:35251-0x1015ce240570000, quorum=127.0.0.1:55007, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-21T10:13:30,429 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3c1f701{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T10:13:30,431 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T10:13:30,431 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T10:13:30,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T10:13:30,431 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/hadoop.log.dir/,STOPPED} 2024-11-21T10:13:30,433 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T10:13:30,433 WARN [BP-1964512914-172.17.0.2-1732183995432 heartbeating to localhost/127.0.0.1:39887 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T10:13:30,434 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T10:13:30,434 WARN [BP-1964512914-172.17.0.2-1732183995432 heartbeating to localhost/127.0.0.1:39887 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1964512914-172.17.0.2-1732183995432 (Datanode Uuid 8bf2038d-9705-4e08-a17d-9c0aeb010e6c) service to localhost/127.0.0.1:39887 2024-11-21T10:13:30,435 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data5/current/BP-1964512914-172.17.0.2-1732183995432 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T10:13:30,435 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data6/current/BP-1964512914-172.17.0.2-1732183995432 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T10:13:30,435 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T10:13:30,437 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@34fcb585{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T10:13:30,438 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T10:13:30,438 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T10:13:30,438 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T10:13:30,438 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/hadoop.log.dir/,STOPPED} 2024-11-21T10:13:30,439 WARN [BP-1964512914-172.17.0.2-1732183995432 heartbeating to localhost/127.0.0.1:39887 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T10:13:30,439 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T10:13:30,439 WARN [BP-1964512914-172.17.0.2-1732183995432 heartbeating to localhost/127.0.0.1:39887 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1964512914-172.17.0.2-1732183995432 (Datanode Uuid c231f160-e49d-4029-975a-f35ca7bca8bc) service to localhost/127.0.0.1:39887 2024-11-21T10:13:30,439 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T10:13:30,440 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data3/current/BP-1964512914-172.17.0.2-1732183995432 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T10:13:30,440 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data4/current/BP-1964512914-172.17.0.2-1732183995432 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T10:13:30,440 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T10:13:30,442 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@79b30fb0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-21T10:13:30,442 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T10:13:30,442 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T10:13:30,442 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T10:13:30,442 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/hadoop.log.dir/,STOPPED} 2024-11-21T10:13:30,443 WARN [BP-1964512914-172.17.0.2-1732183995432 heartbeating to localhost/127.0.0.1:39887 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-21T10:13:30,443 WARN [BP-1964512914-172.17.0.2-1732183995432 heartbeating to localhost/127.0.0.1:39887 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1964512914-172.17.0.2-1732183995432 (Datanode Uuid 908e4323-5680-4328-b07f-2c5ad8591ae9) service to localhost/127.0.0.1:39887 2024-11-21T10:13:30,443 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-21T10:13:30,443 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-21T10:13:30,444 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data1/current/BP-1964512914-172.17.0.2-1732183995432 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T10:13:30,444 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/cluster_19a43f82-92c3-35d1-94fe-da3f28af8c6f/data/data2/current/BP-1964512914-172.17.0.2-1732183995432 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-21T10:13:30,444 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-21T10:13:30,452 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64418798{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-21T10:13:30,452 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-21T10:13:30,453 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-21T10:13:30,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-21T10:13:30,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/45f03af9-862c-6d35-ae89-0fdc3b19ce04/hadoop.log.dir/,STOPPED} 2024-11-21T10:13:30,462 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-21T10:13:30,510 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down