2024-11-09 02:13:14,880 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-09 02:13:14,895 main DEBUG Took 0.013303 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-09 02:13:14,896 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-09 02:13:14,896 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-09 02:13:14,897 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-09 02:13:14,898 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,907 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-09 02:13:14,925 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,927 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,928 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,928 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,929 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,929 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,931 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,931 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,932 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,933 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,934 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,935 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,935 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,936 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,936 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,937 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,938 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,938 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,939 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,939 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,940 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,940 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,941 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-09 02:13:14,942 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,942 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-09 02:13:14,944 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-09 02:13:14,946 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-09 02:13:14,948 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-09 02:13:14,949 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-09 02:13:14,951 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-09 02:13:14,952 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-09 02:13:14,963 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-09 02:13:14,966 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-09 02:13:14,968 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-09 02:13:14,969 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-09 02:13:14,969 main DEBUG createAppenders(={Console}) 2024-11-09 02:13:14,970 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-09 02:13:14,971 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-09 02:13:14,971 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-09 02:13:14,972 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-09 02:13:14,972 main DEBUG OutputStream closed 2024-11-09 02:13:14,973 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-09 02:13:14,973 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-09 02:13:14,974 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-09 02:13:15,059 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-09 02:13:15,062 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-09 02:13:15,064 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-09 02:13:15,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-09 02:13:15,065 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-09 02:13:15,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-09 02:13:15,066 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-09 02:13:15,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-09 02:13:15,067 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-09 02:13:15,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-09 02:13:15,068 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-09 02:13:15,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-09 02:13:15,069 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-09 02:13:15,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-09 02:13:15,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-09 02:13:15,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-09 02:13:15,070 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-09 02:13:15,071 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-09 02:13:15,073 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09 02:13:15,073 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-09 02:13:15,074 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-09 02:13:15,075 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-09T02:13:15,093 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-09 02:13:15,097 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-09 02:13:15,098 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-09T02:13:15,382 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db 2024-11-09T02:13:15,402 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-09T02:13:15,424 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4, deleteOnExit=true 2024-11-09T02:13:15,425 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-09T02:13:15,426 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/test.cache.data in system properties and HBase conf 2024-11-09T02:13:15,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/hadoop.tmp.dir in system properties and HBase conf 2024-11-09T02:13:15,427 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/hadoop.log.dir in system properties and HBase conf 2024-11-09T02:13:15,428 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-09T02:13:15,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-09T02:13:15,429 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-09T02:13:15,517 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-09T02:13:15,610 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-09T02:13:15,614 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-09T02:13:15,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-09T02:13:15,615 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-09T02:13:15,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T02:13:15,616 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-09T02:13:15,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-09T02:13:15,617 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-09T02:13:15,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T02:13:15,618 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-09T02:13:15,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/nfs.dump.dir in system properties and HBase conf 2024-11-09T02:13:15,619 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/java.io.tmpdir in system properties and HBase conf 2024-11-09T02:13:15,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-09T02:13:15,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-09T02:13:15,620 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-09T02:13:16,858 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-09T02:13:16,929 INFO [Time-limited test {}] log.Log(170): Logging initialized @2827ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-09T02:13:17,038 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T02:13:17,112 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T02:13:17,133 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T02:13:17,134 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T02:13:17,135 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-09T02:13:17,147 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T02:13:17,151 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/hadoop.log.dir/,AVAILABLE} 2024-11-09T02:13:17,152 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T02:13:17,346 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@376fa8f3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/java.io.tmpdir/jetty-localhost-32829-hadoop-hdfs-3_4_1-tests_jar-_-any-3080991807656550122/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T02:13:17,357 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:32829} 2024-11-09T02:13:17,357 INFO [Time-limited test {}] server.Server(415): Started @3256ms 2024-11-09T02:13:18,023 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T02:13:18,033 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T02:13:18,034 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T02:13:18,034 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T02:13:18,035 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T02:13:18,035 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/hadoop.log.dir/,AVAILABLE} 2024-11-09T02:13:18,036 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T02:13:18,138 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2226d6a6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/java.io.tmpdir/jetty-localhost-45499-hadoop-hdfs-3_4_1-tests_jar-_-any-4145048946695120525/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T02:13:18,139 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:45499} 2024-11-09T02:13:18,139 INFO [Time-limited test {}] server.Server(415): Started @4038ms 2024-11-09T02:13:18,188 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T02:13:18,294 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T02:13:18,301 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T02:13:18,304 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T02:13:18,305 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T02:13:18,305 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T02:13:18,306 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/hadoop.log.dir/,AVAILABLE} 2024-11-09T02:13:18,307 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T02:13:18,414 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7cfccf27{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/java.io.tmpdir/jetty-localhost-45897-hadoop-hdfs-3_4_1-tests_jar-_-any-14377095618512666077/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T02:13:18,415 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:45897} 2024-11-09T02:13:18,415 INFO [Time-limited test {}] server.Server(415): Started @4314ms 2024-11-09T02:13:18,419 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T02:13:18,455 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-09T02:13:18,460 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-09T02:13:18,462 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-09T02:13:18,463 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-09T02:13:18,463 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-09T02:13:18,464 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/hadoop.log.dir/,AVAILABLE} 2024-11-09T02:13:18,464 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-09T02:13:18,562 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@67ddfd25{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/java.io.tmpdir/jetty-localhost-46413-hadoop-hdfs-3_4_1-tests_jar-_-any-1906569291799767604/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T02:13:18,562 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:46413} 2024-11-09T02:13:18,563 INFO [Time-limited test {}] server.Server(415): Started @4462ms 2024-11-09T02:13:18,564 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-09T02:13:19,831 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data3/current/BP-306821699-172.17.0.3-1731118396170/current, will proceed with Du for space computation calculation, 2024-11-09T02:13:19,832 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data4/current/BP-306821699-172.17.0.3-1731118396170/current, will proceed with Du for space computation calculation, 2024-11-09T02:13:19,836 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data1/current/BP-306821699-172.17.0.3-1731118396170/current, will proceed with Du for space computation calculation, 2024-11-09T02:13:19,839 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data2/current/BP-306821699-172.17.0.3-1731118396170/current, will proceed with Du for space computation calculation, 2024-11-09T02:13:19,896 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T02:13:19,897 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T02:13:19,979 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8879b0a3cebe50f9 with lease ID 0xcf4b07445fc01786: Processing first storage report for DS-5399098e-be1d-4c86-8665-bb7420bd7139 from datanode DatanodeRegistration(127.0.0.1:39093, datanodeUuid=dd851afc-732d-4c4c-b263-dd62f61cbed1, infoPort=41983, infoSecurePort=0, ipcPort=43321, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170) 2024-11-09T02:13:19,981 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8879b0a3cebe50f9 with lease ID 0xcf4b07445fc01786: from storage DS-5399098e-be1d-4c86-8665-bb7420bd7139 node DatanodeRegistration(127.0.0.1:39093, datanodeUuid=dd851afc-732d-4c4c-b263-dd62f61cbed1, infoPort=41983, infoSecurePort=0, ipcPort=43321, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-09T02:13:19,981 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8f1769b7deccd161 with lease ID 0xcf4b07445fc01785: Processing first storage report for DS-1e0d34a9-ddd0-4293-9074-b2273601121a from datanode DatanodeRegistration(127.0.0.1:43193, datanodeUuid=cfe1fdc4-2d17-4683-9ab8-0963e6c7047c, infoPort=35315, infoSecurePort=0, ipcPort=45235, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170) 2024-11-09T02:13:19,982 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8f1769b7deccd161 with lease ID 0xcf4b07445fc01785: from storage DS-1e0d34a9-ddd0-4293-9074-b2273601121a node DatanodeRegistration(127.0.0.1:43193, datanodeUuid=cfe1fdc4-2d17-4683-9ab8-0963e6c7047c, infoPort=35315, infoSecurePort=0, ipcPort=45235, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T02:13:19,982 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8879b0a3cebe50f9 with lease ID 0xcf4b07445fc01786: Processing first storage report for DS-78ec9f1c-b495-416b-a9ae-5d550cc74a16 from datanode DatanodeRegistration(127.0.0.1:39093, datanodeUuid=dd851afc-732d-4c4c-b263-dd62f61cbed1, infoPort=41983, infoSecurePort=0, ipcPort=43321, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170) 2024-11-09T02:13:19,982 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8879b0a3cebe50f9 with lease ID 0xcf4b07445fc01786: from storage DS-78ec9f1c-b495-416b-a9ae-5d550cc74a16 node DatanodeRegistration(127.0.0.1:39093, datanodeUuid=dd851afc-732d-4c4c-b263-dd62f61cbed1, infoPort=41983, infoSecurePort=0, ipcPort=43321, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T02:13:19,982 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8f1769b7deccd161 with lease ID 0xcf4b07445fc01785: Processing first storage report for DS-25ad651d-09fa-4f3d-9f89-bdafc30994b0 from datanode DatanodeRegistration(127.0.0.1:43193, datanodeUuid=cfe1fdc4-2d17-4683-9ab8-0963e6c7047c, infoPort=35315, infoSecurePort=0, ipcPort=45235, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170) 2024-11-09T02:13:19,983 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8f1769b7deccd161 with lease ID 0xcf4b07445fc01785: from storage DS-25ad651d-09fa-4f3d-9f89-bdafc30994b0 node DatanodeRegistration(127.0.0.1:43193, datanodeUuid=cfe1fdc4-2d17-4683-9ab8-0963e6c7047c, infoPort=35315, infoSecurePort=0, ipcPort=45235, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T02:13:20,020 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data5/current/BP-306821699-172.17.0.3-1731118396170/current, will proceed with Du for space computation calculation, 2024-11-09T02:13:20,020 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data6/current/BP-306821699-172.17.0.3-1731118396170/current, will proceed with Du for space computation calculation, 2024-11-09T02:13:20,055 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-09T02:13:20,066 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3e33054522b4d4a3 with lease ID 0xcf4b07445fc01787: Processing first storage report for DS-04930e57-9ab7-4098-82a3-72b98471424f from datanode DatanodeRegistration(127.0.0.1:45399, datanodeUuid=c5bbc037-aae6-45bb-9ee0-c0a8ee5d9ef6, infoPort=42707, infoSecurePort=0, ipcPort=38645, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170) 2024-11-09T02:13:20,067 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3e33054522b4d4a3 with lease ID 0xcf4b07445fc01787: from storage DS-04930e57-9ab7-4098-82a3-72b98471424f node DatanodeRegistration(127.0.0.1:45399, datanodeUuid=c5bbc037-aae6-45bb-9ee0-c0a8ee5d9ef6, infoPort=42707, infoSecurePort=0, ipcPort=38645, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-09T02:13:20,067 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x3e33054522b4d4a3 with lease ID 0xcf4b07445fc01787: Processing first storage report for DS-ac1fdd67-16c1-4e59-a0cc-4621f1a40fdb from datanode DatanodeRegistration(127.0.0.1:45399, datanodeUuid=c5bbc037-aae6-45bb-9ee0-c0a8ee5d9ef6, infoPort=42707, infoSecurePort=0, ipcPort=38645, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170) 2024-11-09T02:13:20,067 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x3e33054522b4d4a3 with lease ID 0xcf4b07445fc01787: from storage DS-ac1fdd67-16c1-4e59-a0cc-4621f1a40fdb node DatanodeRegistration(127.0.0.1:45399, datanodeUuid=c5bbc037-aae6-45bb-9ee0-c0a8ee5d9ef6, infoPort=42707, infoSecurePort=0, ipcPort=38645, storageInfo=lv=-57;cid=testClusterID;nsid=870153938;c=1731118396170), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-09T02:13:20,090 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db 2024-11-09T02:13:20,200 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/zookeeper_0, clientPort=59275, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-09T02:13:20,217 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59275 2024-11-09T02:13:20,237 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:13:20,243 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:13:20,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741825_1001 (size=7) 2024-11-09T02:13:20,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741825_1001 (size=7) 2024-11-09T02:13:20,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741825_1001 (size=7) 2024-11-09T02:13:20,977 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 with version=8 2024-11-09T02:13:20,978 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/hbase-staging 2024-11-09T02:13:21,129 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-09T02:13:21,548 INFO [Time-limited test {}] client.ConnectionUtils(128): master/53f286d3af3b:0 server-side Connection retries=45 2024-11-09T02:13:21,560 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:21,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:21,567 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T02:13:21,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:21,567 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T02:13:21,744 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-09T02:13:21,822 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-09T02:13:21,831 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-09T02:13:21,834 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T02:13:21,860 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 3173 (auto-detected) 2024-11-09T02:13:21,861 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-09T02:13:21,888 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:36011 2024-11-09T02:13:21,918 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36011 connecting to ZooKeeper ensemble=127.0.0.1:59275 2024-11-09T02:13:22,057 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:360110x0, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T02:13:22,061 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36011-0x1011d5e7e540000 connected 2024-11-09T02:13:22,179 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:13:22,181 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:13:22,192 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:13:22,196 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8, hbase.cluster.distributed=false 2024-11-09T02:13:22,223 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T02:13:22,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36011 2024-11-09T02:13:22,229 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36011 2024-11-09T02:13:22,230 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36011 2024-11-09T02:13:22,231 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36011 2024-11-09T02:13:22,232 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36011 2024-11-09T02:13:22,363 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/53f286d3af3b:0 server-side Connection retries=45 2024-11-09T02:13:22,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:22,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:22,365 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T02:13:22,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:22,365 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T02:13:22,368 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T02:13:22,371 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T02:13:22,373 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45941 2024-11-09T02:13:22,375 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45941 connecting to ZooKeeper ensemble=127.0.0.1:59275 2024-11-09T02:13:22,376 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:13:22,381 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:13:22,398 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:459410x0, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T02:13:22,399 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:13:22,399 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45941-0x1011d5e7e540001 connected 2024-11-09T02:13:22,404 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T02:13:22,414 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T02:13:22,417 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T02:13:22,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T02:13:22,423 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45941 2024-11-09T02:13:22,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45941 2024-11-09T02:13:22,424 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45941 2024-11-09T02:13:22,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45941 2024-11-09T02:13:22,426 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45941 2024-11-09T02:13:22,442 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/53f286d3af3b:0 server-side Connection retries=45 2024-11-09T02:13:22,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:22,442 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:22,443 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T02:13:22,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:22,443 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T02:13:22,443 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T02:13:22,444 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T02:13:22,444 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46883 2024-11-09T02:13:22,446 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46883 connecting to ZooKeeper ensemble=127.0.0.1:59275 2024-11-09T02:13:22,447 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:13:22,451 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:13:22,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:468830x0, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T02:13:22,462 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:468830x0, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:13:22,462 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46883-0x1011d5e7e540002 connected 2024-11-09T02:13:22,462 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T02:13:22,464 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T02:13:22,465 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T02:13:22,467 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T02:13:22,468 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46883 2024-11-09T02:13:22,469 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46883 2024-11-09T02:13:22,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46883 2024-11-09T02:13:22,472 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46883 2024-11-09T02:13:22,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46883 2024-11-09T02:13:22,490 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/53f286d3af3b:0 server-side Connection retries=45 2024-11-09T02:13:22,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:22,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:22,490 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-09T02:13:22,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-09T02:13:22,491 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-09T02:13:22,491 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-09T02:13:22,491 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-09T02:13:22,492 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33749 2024-11-09T02:13:22,494 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33749 connecting to ZooKeeper ensemble=127.0.0.1:59275 2024-11-09T02:13:22,495 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:13:22,498 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:13:22,514 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:337490x0, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-09T02:13:22,515 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:13:22,515 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33749-0x1011d5e7e540003 connected 2024-11-09T02:13:22,515 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-09T02:13:22,517 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-09T02:13:22,518 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-09T02:13:22,521 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-09T02:13:22,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33749 2024-11-09T02:13:22,523 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33749 2024-11-09T02:13:22,524 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33749 2024-11-09T02:13:22,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33749 2024-11-09T02:13:22,525 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33749 2024-11-09T02:13:22,545 DEBUG [M:0;53f286d3af3b:36011 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;53f286d3af3b:36011 2024-11-09T02:13:22,546 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/53f286d3af3b,36011,1731118401306 2024-11-09T02:13:22,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:13:22,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:13:22,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:13:22,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:13:22,567 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/53f286d3af3b,36011,1731118401306 2024-11-09T02:13:22,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T02:13:22,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T02:13:22,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-09T02:13:22,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:22,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:22,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:22,598 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:22,599 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-09T02:13:22,601 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/53f286d3af3b,36011,1731118401306 from backup master directory 2024-11-09T02:13:22,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:13:22,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/53f286d3af3b,36011,1731118401306 2024-11-09T02:13:22,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:13:22,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:13:22,609 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-09T02:13:22,609 WARN [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T02:13:22,610 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=53f286d3af3b,36011,1731118401306 2024-11-09T02:13:22,612 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-09T02:13:22,614 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-09T02:13:22,672 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/hbase.id] with ID: 016251ff-734f-4ad4-acfd-bd7e0379d62f 2024-11-09T02:13:22,672 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/.tmp/hbase.id 2024-11-09T02:13:22,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741826_1002 (size=42) 2024-11-09T02:13:22,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741826_1002 (size=42) 2024-11-09T02:13:22,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741826_1002 (size=42) 2024-11-09T02:13:22,687 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/.tmp/hbase.id]:[hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/hbase.id] 2024-11-09T02:13:22,735 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-09T02:13:22,740 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-09T02:13:22,759 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 17ms. 2024-11-09T02:13:22,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:22,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:22,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:22,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:22,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741827_1003 (size=196) 2024-11-09T02:13:22,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741827_1003 (size=196) 2024-11-09T02:13:22,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741827_1003 (size=196) 2024-11-09T02:13:22,810 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:13:22,812 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-09T02:13:22,825 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:13:22,829 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:13:22,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741828_1004 (size=1189) 2024-11-09T02:13:22,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741828_1004 (size=1189) 2024-11-09T02:13:22,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741828_1004 (size=1189) 2024-11-09T02:13:22,877 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store 2024-11-09T02:13:22,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741829_1005 (size=34) 2024-11-09T02:13:22,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741829_1005 (size=34) 2024-11-09T02:13:22,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741829_1005 (size=34) 2024-11-09T02:13:22,906 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-09T02:13:22,909 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:22,911 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T02:13:22,911 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:13:22,911 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:13:22,913 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T02:13:22,913 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:13:22,913 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:13:22,914 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731118402910Disabling compacts and flushes for region at 1731118402910Disabling writes for close at 1731118402913 (+3 ms)Writing region close event to WAL at 1731118402913Closed at 1731118402913 2024-11-09T02:13:22,916 WARN [master/53f286d3af3b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/.initializing 2024-11-09T02:13:22,917 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/WALs/53f286d3af3b,36011,1731118401306 2024-11-09T02:13:22,925 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T02:13:22,939 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=53f286d3af3b%2C36011%2C1731118401306, suffix=, logDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/WALs/53f286d3af3b,36011,1731118401306, archiveDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/oldWALs, maxLogs=10 2024-11-09T02:13:22,961 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/WALs/53f286d3af3b,36011,1731118401306/53f286d3af3b%2C36011%2C1731118401306.1731118402944, exclude list is [], retry=0 2024-11-09T02:13:22,964 WARN [IPC Server handler 2 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:22,965 WARN [IPC Server handler 2 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:22,965 WARN [IPC Server handler 2 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:22,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43193,DS-1e0d34a9-ddd0-4293-9074-b2273601121a,DISK] 2024-11-09T02:13:22,979 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45399,DS-04930e57-9ab7-4098-82a3-72b98471424f,DISK] 2024-11-09T02:13:22,983 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-09T02:13:23,020 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/WALs/53f286d3af3b,36011,1731118401306/53f286d3af3b%2C36011%2C1731118401306.1731118402944 2024-11-09T02:13:23,022 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35315:35315),(127.0.0.1/127.0.0.1:42707:42707)] 2024-11-09T02:13:23,023 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:13:23,023 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:23,027 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,028 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,069 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,096 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-09T02:13:23,100 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:23,103 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:23,104 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,108 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-09T02:13:23,108 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:23,109 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:23,110 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,113 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-09T02:13:23,114 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:23,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:23,115 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,119 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-09T02:13:23,119 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:23,121 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:23,121 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,126 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,127 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,134 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,135 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,141 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T02:13:23,147 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-09T02:13:23,154 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:23,156 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68642490, jitterRate=0.022852808237075806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T02:13:23,167 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731118403042Initializing all the Stores at 1731118403044 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118403045 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118403045Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118403046 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118403046Cleaning up temporary data from old regions at 1731118403135 (+89 ms)Region opened successfully at 1731118403167 (+32 ms) 2024-11-09T02:13:23,169 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-09T02:13:23,209 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc73dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=53f286d3af3b/172.17.0.3:0 2024-11-09T02:13:23,242 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-09T02:13:23,253 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:23,253 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:23,256 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:23,257 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T02:13:23,262 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T02:13:23,262 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-09T02:13:23,288 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-09T02:13:23,297 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-09T02:13:23,345 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-09T02:13:23,350 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-09T02:13:23,352 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-09T02:13:23,363 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-09T02:13:23,365 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-09T02:13:23,369 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-09T02:13:23,376 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-09T02:13:23,378 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-09T02:13:23,387 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-09T02:13:23,408 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-09T02:13:23,419 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-09T02:13:23,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T02:13:23,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T02:13:23,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T02:13:23,429 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-09T02:13:23,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,434 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=53f286d3af3b,36011,1731118401306, sessionid=0x1011d5e7e540000, setting cluster-up flag (Was=false) 2024-11-09T02:13:23,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,493 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-09T02:13:23,497 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=53f286d3af3b,36011,1731118401306 2024-11-09T02:13:23,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,524 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:23,556 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-09T02:13:23,559 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=53f286d3af3b,36011,1731118401306 2024-11-09T02:13:23,565 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-09T02:13:23,630 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(746): ClusterId : 016251ff-734f-4ad4-acfd-bd7e0379d62f 2024-11-09T02:13:23,630 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(746): ClusterId : 016251ff-734f-4ad4-acfd-bd7e0379d62f 2024-11-09T02:13:23,632 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.HRegionServer(746): ClusterId : 016251ff-734f-4ad4-acfd-bd7e0379d62f 2024-11-09T02:13:23,633 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T02:13:23,633 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T02:13:23,633 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-09T02:13:23,632 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:23,642 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-09T02:13:23,648 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-09T02:13:23,653 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 53f286d3af3b,36011,1731118401306 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-09T02:13:23,659 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T02:13:23,659 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T02:13:23,659 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-09T02:13:23,660 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T02:13:23,660 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T02:13:23,660 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-09T02:13:23,661 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/53f286d3af3b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:13:23,661 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/53f286d3af3b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:13:23,661 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/53f286d3af3b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:13:23,661 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/53f286d3af3b:0, corePoolSize=5, maxPoolSize=5 2024-11-09T02:13:23,661 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/53f286d3af3b:0, corePoolSize=10, maxPoolSize=10 2024-11-09T02:13:23,661 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:23,661 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/53f286d3af3b:0, corePoolSize=2, maxPoolSize=2 2024-11-09T02:13:23,661 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:23,662 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118433662 2024-11-09T02:13:23,663 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-09T02:13:23,664 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-09T02:13:23,667 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:23,668 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-09T02:13:23,668 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-09T02:13:23,668 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-09T02:13:23,668 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-09T02:13:23,668 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-09T02:13:23,669 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:23,672 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-09T02:13:23,674 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-09T02:13:23,674 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-09T02:13:23,674 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:23,675 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-09T02:13:23,676 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-09T02:13:23,677 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-09T02:13:23,679 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.large.0-1731118403678,5,FailOnTimeoutGroup] 2024-11-09T02:13:23,680 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.small.0-1731118403679,5,FailOnTimeoutGroup] 2024-11-09T02:13:23,680 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:23,680 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-09T02:13:23,682 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:23,682 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:23,685 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T02:13:23,685 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T02:13:23,685 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-09T02:13:23,685 DEBUG [RS:1;53f286d3af3b:46883 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bb7bc67, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=53f286d3af3b/172.17.0.3:0 2024-11-09T02:13:23,685 DEBUG [RS:0;53f286d3af3b:45941 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55fc04a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=53f286d3af3b/172.17.0.3:0 2024-11-09T02:13:23,685 DEBUG [RS:2;53f286d3af3b:33749 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ffcf52e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=53f286d3af3b/172.17.0.3:0 2024-11-09T02:13:23,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741831_1007 (size=1321) 2024-11-09T02:13:23,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741831_1007 (size=1321) 2024-11-09T02:13:23,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741831_1007 (size=1321) 2024-11-09T02:13:23,704 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-09T02:13:23,705 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:23,707 DEBUG [RS:0;53f286d3af3b:45941 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;53f286d3af3b:45941 2024-11-09T02:13:23,707 DEBUG [RS:1;53f286d3af3b:46883 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;53f286d3af3b:46883 2024-11-09T02:13:23,710 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T02:13:23,710 DEBUG [RS:2;53f286d3af3b:33749 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;53f286d3af3b:33749 2024-11-09T02:13:23,710 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T02:13:23,711 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T02:13:23,711 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T02:13:23,711 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-09T02:13:23,711 DEBUG [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T02:13:23,711 DEBUG [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T02:13:23,711 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-09T02:13:23,711 DEBUG [RS:2;53f286d3af3b:33749 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-09T02:13:23,714 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(2659): reportForDuty to master=53f286d3af3b,36011,1731118401306 with port=46883, startcode=1731118402441 2024-11-09T02:13:23,714 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.HRegionServer(2659): reportForDuty to master=53f286d3af3b,36011,1731118401306 with port=33749, startcode=1731118402489 2024-11-09T02:13:23,714 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(2659): reportForDuty to master=53f286d3af3b,36011,1731118401306 with port=45941, startcode=1731118402318 2024-11-09T02:13:23,727 DEBUG [RS:2;53f286d3af3b:33749 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T02:13:23,727 DEBUG [RS:1;53f286d3af3b:46883 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T02:13:23,727 DEBUG [RS:0;53f286d3af3b:45941 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-09T02:13:23,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741832_1008 (size=32) 2024-11-09T02:13:23,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741832_1008 (size=32) 2024-11-09T02:13:23,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741832_1008 (size=32) 2024-11-09T02:13:23,731 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:23,734 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T02:13:23,737 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T02:13:23,738 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:23,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:23,739 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T02:13:23,744 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T02:13:23,744 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:23,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:23,745 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T02:13:23,749 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T02:13:23,749 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:23,750 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:23,751 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T02:13:23,757 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T02:13:23,757 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:23,758 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:23,759 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T02:13:23,761 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740 2024-11-09T02:13:23,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740 2024-11-09T02:13:23,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T02:13:23,766 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T02:13:23,767 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T02:13:23,770 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39243, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T02:13:23,770 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37601, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T02:13:23,771 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39425, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-09T02:13:23,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T02:13:23,778 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:23,778 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 53f286d3af3b,33749,1731118402489 2024-11-09T02:13:23,779 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64366895, jitterRate=-0.040858522057533264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T02:13:23,781 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] master.ServerManager(517): Registering regionserver=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:23,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731118403731Initializing all the Stores at 1731118403733 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118403733Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118403734 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118403734Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118403734Cleaning up temporary data from old regions at 1731118403766 (+32 ms)Region opened successfully at 1731118403782 (+16 ms) 2024-11-09T02:13:23,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T02:13:23,783 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T02:13:23,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T02:13:23,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T02:13:23,783 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T02:13:23,784 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T02:13:23,784 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731118403783Disabling compacts and flushes for region at 1731118403783Disabling writes for close at 1731118403783Writing region close event to WAL at 1731118403784 (+1 ms)Closed at 1731118403784 2024-11-09T02:13:23,788 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:23,788 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-09T02:13:23,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T02:13:23,795 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 53f286d3af3b,46883,1731118402441 2024-11-09T02:13:23,795 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] master.ServerManager(517): Registering regionserver=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:23,798 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 53f286d3af3b,45941,1731118402318 2024-11-09T02:13:23,799 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] master.ServerManager(517): Registering regionserver=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:23,799 DEBUG [RS:2;53f286d3af3b:33749 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:23,799 DEBUG [RS:2;53f286d3af3b:33749 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45553 2024-11-09T02:13:23,799 DEBUG [RS:2;53f286d3af3b:33749 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T02:13:23,801 DEBUG [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:23,801 DEBUG [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45553 2024-11-09T02:13:23,801 DEBUG [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T02:13:23,805 DEBUG [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:23,805 DEBUG [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45553 2024-11-09T02:13:23,805 DEBUG [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-09T02:13:23,808 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:23,811 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-09T02:13:23,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T02:13:23,963 WARN [53f286d3af3b:36011 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-09T02:13:23,978 DEBUG [RS:2;53f286d3af3b:33749 {}] zookeeper.ZKUtil(111): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/53f286d3af3b,33749,1731118402489 2024-11-09T02:13:23,978 DEBUG [RS:1;53f286d3af3b:46883 {}] zookeeper.ZKUtil(111): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/53f286d3af3b,46883,1731118402441 2024-11-09T02:13:23,978 WARN [RS:2;53f286d3af3b:33749 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T02:13:23,978 WARN [RS:1;53f286d3af3b:46883 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T02:13:23,979 INFO [RS:2;53f286d3af3b:33749 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:13:23,979 INFO [RS:1;53f286d3af3b:46883 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:13:23,979 DEBUG [RS:2;53f286d3af3b:33749 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489 2024-11-09T02:13:23,979 DEBUG [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,46883,1731118402441 2024-11-09T02:13:23,979 DEBUG [RS:0;53f286d3af3b:45941 {}] zookeeper.ZKUtil(111): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/53f286d3af3b,45941,1731118402318 2024-11-09T02:13:23,980 WARN [RS:0;53f286d3af3b:45941 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-09T02:13:23,980 INFO [RS:0;53f286d3af3b:45941 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:13:23,980 DEBUG [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,45941,1731118402318 2024-11-09T02:13:23,982 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [53f286d3af3b,33749,1731118402489] 2024-11-09T02:13:23,983 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [53f286d3af3b,46883,1731118402441] 2024-11-09T02:13:23,983 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [53f286d3af3b,45941,1731118402318] 2024-11-09T02:13:24,006 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T02:13:24,006 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T02:13:24,006 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-09T02:13:24,022 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T02:13:24,022 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T02:13:24,022 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-09T02:13:24,027 INFO [RS:1;53f286d3af3b:46883 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T02:13:24,027 INFO [RS:2;53f286d3af3b:33749 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T02:13:24,027 INFO [RS:0;53f286d3af3b:45941 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-09T02:13:24,027 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,027 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,027 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,028 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T02:13:24,028 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T02:13:24,028 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-09T02:13:24,034 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T02:13:24,034 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T02:13:24,034 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-09T02:13:24,036 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,036 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,036 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,036 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,036 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,036 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,036 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,036 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,036 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,036 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,036 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,036 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0, corePoolSize=2, maxPoolSize=2 2024-11-09T02:13:24,037 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0, corePoolSize=2, maxPoolSize=2 2024-11-09T02:13:24,037 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0, corePoolSize=2, maxPoolSize=2 2024-11-09T02:13:24,037 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,037 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,038 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,038 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,038 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,038 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,038 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,038 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/53f286d3af3b:0, corePoolSize=1, maxPoolSize=1 2024-11-09T02:13:24,038 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/53f286d3af3b:0, corePoolSize=3, maxPoolSize=3 2024-11-09T02:13:24,038 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/53f286d3af3b:0, corePoolSize=3, maxPoolSize=3 2024-11-09T02:13:24,038 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/53f286d3af3b:0, corePoolSize=3, maxPoolSize=3 2024-11-09T02:13:24,038 DEBUG [RS:2;53f286d3af3b:33749 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0, corePoolSize=3, maxPoolSize=3 2024-11-09T02:13:24,038 DEBUG [RS:1;53f286d3af3b:46883 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0, corePoolSize=3, maxPoolSize=3 2024-11-09T02:13:24,038 DEBUG [RS:0;53f286d3af3b:45941 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/53f286d3af3b:0, corePoolSize=3, maxPoolSize=3 2024-11-09T02:13:24,040 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,040 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,041 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,33749,1731118402489-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T02:13:24,041 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,45941,1731118402318-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T02:13:24,041 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,041 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,46883,1731118402441-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T02:13:24,062 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T02:13:24,065 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T02:13:24,065 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-09T02:13:24,065 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,33749,1731118402489-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,065 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,45941,1731118402318-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,065 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,46883,1731118402441-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,065 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,065 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,065 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,065 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.Replication(171): 53f286d3af3b,45941,1731118402318 started 2024-11-09T02:13:24,065 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.Replication(171): 53f286d3af3b,46883,1731118402441 started 2024-11-09T02:13:24,065 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.Replication(171): 53f286d3af3b,33749,1731118402489 started 2024-11-09T02:13:24,091 INFO [RS:2;53f286d3af3b:33749 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,091 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,091 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:24,091 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.HRegionServer(1482): Serving as 53f286d3af3b,33749,1731118402489, RpcServer on 53f286d3af3b/172.17.0.3:33749, sessionid=0x1011d5e7e540003 2024-11-09T02:13:24,091 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(1482): Serving as 53f286d3af3b,46883,1731118402441, RpcServer on 53f286d3af3b/172.17.0.3:46883, sessionid=0x1011d5e7e540002 2024-11-09T02:13:24,091 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(1482): Serving as 53f286d3af3b,45941,1731118402318, RpcServer on 53f286d3af3b/172.17.0.3:45941, sessionid=0x1011d5e7e540001 2024-11-09T02:13:24,092 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T02:13:24,092 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T02:13:24,092 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-09T02:13:24,092 DEBUG [RS:2;53f286d3af3b:33749 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 53f286d3af3b,33749,1731118402489 2024-11-09T02:13:24,092 DEBUG [RS:1;53f286d3af3b:46883 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 53f286d3af3b,46883,1731118402441 2024-11-09T02:13:24,092 DEBUG [RS:0;53f286d3af3b:45941 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 53f286d3af3b,45941,1731118402318 2024-11-09T02:13:24,092 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '53f286d3af3b,45941,1731118402318' 2024-11-09T02:13:24,092 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '53f286d3af3b,46883,1731118402441' 2024-11-09T02:13:24,092 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '53f286d3af3b,33749,1731118402489' 2024-11-09T02:13:24,093 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T02:13:24,093 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T02:13:24,093 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-09T02:13:24,094 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T02:13:24,094 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T02:13:24,094 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-09T02:13:24,094 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T02:13:24,095 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T02:13:24,095 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T02:13:24,095 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T02:13:24,095 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-09T02:13:24,095 DEBUG [RS:2;53f286d3af3b:33749 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 53f286d3af3b,33749,1731118402489 2024-11-09T02:13:24,095 DEBUG [RS:1;53f286d3af3b:46883 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 53f286d3af3b,46883,1731118402441 2024-11-09T02:13:24,095 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-09T02:13:24,095 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '53f286d3af3b,46883,1731118402441' 2024-11-09T02:13:24,095 DEBUG [RS:0;53f286d3af3b:45941 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 53f286d3af3b,45941,1731118402318 2024-11-09T02:13:24,095 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '53f286d3af3b,33749,1731118402489' 2024-11-09T02:13:24,095 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T02:13:24,095 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '53f286d3af3b,45941,1731118402318' 2024-11-09T02:13:24,095 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T02:13:24,095 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-09T02:13:24,096 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T02:13:24,096 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T02:13:24,096 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-09T02:13:24,096 DEBUG [RS:2;53f286d3af3b:33749 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T02:13:24,096 INFO [RS:2;53f286d3af3b:33749 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T02:13:24,097 INFO [RS:2;53f286d3af3b:33749 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T02:13:24,097 DEBUG [RS:1;53f286d3af3b:46883 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T02:13:24,097 DEBUG [RS:0;53f286d3af3b:45941 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-09T02:13:24,097 INFO [RS:0;53f286d3af3b:45941 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T02:13:24,097 INFO [RS:1;53f286d3af3b:46883 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-09T02:13:24,097 INFO [RS:0;53f286d3af3b:45941 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T02:13:24,097 INFO [RS:1;53f286d3af3b:46883 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-09T02:13:24,208 INFO [RS:1;53f286d3af3b:46883 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T02:13:24,208 INFO [RS:0;53f286d3af3b:45941 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T02:13:24,209 INFO [RS:2;53f286d3af3b:33749 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-09T02:13:24,212 INFO [RS:0;53f286d3af3b:45941 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=53f286d3af3b%2C45941%2C1731118402318, suffix=, logDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,45941,1731118402318, archiveDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/oldWALs, maxLogs=32 2024-11-09T02:13:24,212 INFO [RS:1;53f286d3af3b:46883 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=53f286d3af3b%2C46883%2C1731118402441, suffix=, logDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,46883,1731118402441, archiveDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/oldWALs, maxLogs=32 2024-11-09T02:13:24,213 INFO [RS:2;53f286d3af3b:33749 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=53f286d3af3b%2C33749%2C1731118402489, suffix=, logDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489, archiveDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/oldWALs, maxLogs=32 2024-11-09T02:13:24,231 DEBUG [RS:0;53f286d3af3b:45941 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,45941,1731118402318/53f286d3af3b%2C45941%2C1731118402318.1731118404218, exclude list is [], retry=0 2024-11-09T02:13:24,231 DEBUG [RS:1;53f286d3af3b:46883 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,46883,1731118402441/53f286d3af3b%2C46883%2C1731118402441.1731118404218, exclude list is [], retry=0 2024-11-09T02:13:24,232 DEBUG [RS:2;53f286d3af3b:33749 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489/53f286d3af3b%2C33749%2C1731118402489.1731118404218, exclude list is [], retry=0 2024-11-09T02:13:24,235 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:24,235 WARN [IPC Server handler 3 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:24,235 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:24,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45399,DS-04930e57-9ab7-4098-82a3-72b98471424f,DISK] 2024-11-09T02:13:24,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45399,DS-04930e57-9ab7-4098-82a3-72b98471424f,DISK] 2024-11-09T02:13:24,236 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43193,DS-1e0d34a9-ddd0-4293-9074-b2273601121a,DISK] 2024-11-09T02:13:24,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39093,DS-5399098e-be1d-4c86-8665-bb7420bd7139,DISK] 2024-11-09T02:13:24,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39093,DS-5399098e-be1d-4c86-8665-bb7420bd7139,DISK] 2024-11-09T02:13:24,237 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43193,DS-1e0d34a9-ddd0-4293-9074-b2273601121a,DISK] 2024-11-09T02:13:24,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39093,DS-5399098e-be1d-4c86-8665-bb7420bd7139,DISK] 2024-11-09T02:13:24,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43193,DS-1e0d34a9-ddd0-4293-9074-b2273601121a,DISK] 2024-11-09T02:13:24,283 INFO [RS:1;53f286d3af3b:46883 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,46883,1731118402441/53f286d3af3b%2C46883%2C1731118402441.1731118404218 2024-11-09T02:13:24,284 DEBUG [RS:1;53f286d3af3b:46883 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42707:42707),(127.0.0.1/127.0.0.1:41983:41983),(127.0.0.1/127.0.0.1:35315:35315)] 2024-11-09T02:13:24,286 INFO [RS:2;53f286d3af3b:33749 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489/53f286d3af3b%2C33749%2C1731118402489.1731118404218 2024-11-09T02:13:24,286 DEBUG [RS:2;53f286d3af3b:33749 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41983:41983),(127.0.0.1/127.0.0.1:35315:35315)] 2024-11-09T02:13:24,286 INFO [RS:0;53f286d3af3b:45941 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,45941,1731118402318/53f286d3af3b%2C45941%2C1731118402318.1731118404218 2024-11-09T02:13:24,287 DEBUG [RS:0;53f286d3af3b:45941 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42707:42707),(127.0.0.1/127.0.0.1:35315:35315),(127.0.0.1/127.0.0.1:41983:41983)] 2024-11-09T02:13:24,468 DEBUG [53f286d3af3b:36011 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-09T02:13:24,478 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(204): Hosts are {53f286d3af3b=0} racks are {/default-rack=0} 2024-11-09T02:13:24,484 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T02:13:24,484 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T02:13:24,484 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-09T02:13:24,484 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T02:13:24,484 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T02:13:24,484 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-09T02:13:24,484 INFO [53f286d3af3b:36011 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T02:13:24,484 INFO [53f286d3af3b:36011 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T02:13:24,484 INFO [53f286d3af3b:36011 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-09T02:13:24,484 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T02:13:24,491 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:24,497 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,33749,1731118402489, state=OPENING 2024-11-09T02:13:24,552 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-09T02:13:24,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:24,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:24,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:24,567 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:24,569 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:24,569 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:24,569 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:24,570 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:24,572 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:24,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,33749,1731118402489}] 2024-11-09T02:13:24,747 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T02:13:24,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50559, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T02:13:24,759 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T02:13:24,759 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:13:24,760 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T02:13:24,763 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=53f286d3af3b%2C33749%2C1731118402489.meta, suffix=.meta, logDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489, archiveDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/oldWALs, maxLogs=32 2024-11-09T02:13:24,778 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, exclude list is [], retry=0 2024-11-09T02:13:24,780 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:24,780 WARN [IPC Server handler 3 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:24,780 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:24,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39093,DS-5399098e-be1d-4c86-8665-bb7420bd7139,DISK] 2024-11-09T02:13:24,782 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43193,DS-1e0d34a9-ddd0-4293-9074-b2273601121a,DISK] 2024-11-09T02:13:24,785 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta 2024-11-09T02:13:24,785 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41983:41983),(127.0.0.1/127.0.0.1:35315:35315)] 2024-11-09T02:13:24,785 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:13:24,787 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T02:13:24,789 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T02:13:24,794 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T02:13:24,798 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T02:13:24,798 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:24,799 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T02:13:24,799 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T02:13:24,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T02:13:24,804 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T02:13:24,804 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:24,805 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:24,805 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T02:13:24,806 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T02:13:24,806 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:24,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:24,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T02:13:24,809 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T02:13:24,809 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:24,810 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:24,810 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T02:13:24,811 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T02:13:24,811 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:24,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:24,812 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T02:13:24,814 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740 2024-11-09T02:13:24,816 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740 2024-11-09T02:13:24,819 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T02:13:24,819 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T02:13:24,819 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T02:13:24,822 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T02:13:24,824 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65515130, jitterRate=-0.0237484872341156}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T02:13:24,824 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T02:13:24,826 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731118404799Writing region info on filesystem at 1731118404799Initializing all the Stores at 1731118404801 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118404802 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118404802Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118404802Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118404802Cleaning up temporary data from old regions at 1731118404819 (+17 ms)Running coprocessor post-open hooks at 1731118404824 (+5 ms)Region opened successfully at 1731118404826 (+2 ms) 2024-11-09T02:13:24,834 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731118404740 2024-11-09T02:13:24,844 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T02:13:24,845 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T02:13:24,847 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:24,850 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,33749,1731118402489, state=OPEN 2024-11-09T02:13:24,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:24,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:24,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:24,899 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:24,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:24,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:24,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:24,900 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:24,900 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:24,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-09T02:13:24,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,33749,1731118402489 in 326 msec 2024-11-09T02:13:24,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-09T02:13:24,913 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.1130 sec 2024-11-09T02:13:24,915 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:24,915 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-09T02:13:24,936 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:13:24,938 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1] 2024-11-09T02:13:24,959 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:13:24,962 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:32789, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:13:24,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.3850 sec 2024-11-09T02:13:24,980 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731118404980, completionTime=-1 2024-11-09T02:13:24,982 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-09T02:13:24,982 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:25,006 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-09T02:13:25,006 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118465006 2024-11-09T02:13:25,006 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118525006 2024-11-09T02:13:25,006 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 24 msec 2024-11-09T02:13:25,008 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:25,014 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,36011,1731118401306-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:25,015 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,36011,1731118401306-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:25,015 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,36011,1731118401306-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:25,016 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-53f286d3af3b:36011, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:25,016 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:25,017 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:25,024 DEBUG [master/53f286d3af3b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-09T02:13:25,051 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.441sec 2024-11-09T02:13:25,052 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-09T02:13:25,053 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-09T02:13:25,054 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-09T02:13:25,055 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-09T02:13:25,055 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-09T02:13:25,056 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,36011,1731118401306-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-09T02:13:25,056 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,36011,1731118401306-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-09T02:13:25,060 DEBUG [master/53f286d3af3b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-09T02:13:25,061 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-09T02:13:25,061 INFO [master/53f286d3af3b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=53f286d3af3b,36011,1731118401306-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-09T02:13:25,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33307714, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:13:25,152 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-09T02:13:25,153 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-09T02:13:25,155 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 53f286d3af3b,36011,-1 for getting cluster id 2024-11-09T02:13:25,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-09T02:13:25,166 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '016251ff-734f-4ad4-acfd-bd7e0379d62f' 2024-11-09T02:13:25,168 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-09T02:13:25,169 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "016251ff-734f-4ad4-acfd-bd7e0379d62f" 2024-11-09T02:13:25,169 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38e78b42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:13:25,169 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [53f286d3af3b,36011,-1] 2024-11-09T02:13:25,172 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-09T02:13:25,175 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:13:25,175 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48424, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-09T02:13:25,179 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cc0e1f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-09T02:13:25,179 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:13:25,186 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1] 2024-11-09T02:13:25,187 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:13:25,189 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:13:25,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=53f286d3af3b,36011,1731118401306 2024-11-09T02:13:25,231 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-09T02:13:25,237 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 53f286d3af3b,36011,1731118401306 2024-11-09T02:13:25,241 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@214aca81 2024-11-09T02:13:25,242 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-09T02:13:25,245 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48436, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-09T02:13:25,256 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-09T02:13:25,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:25,267 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-09T02:13:25,269 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-09T02:13:25,269 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:25,272 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-09T02:13:25,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T02:13:25,280 WARN [IPC Server handler 0 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,280 WARN [IPC Server handler 0 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,280 WARN [IPC Server handler 0 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741837_1013 (size=358) 2024-11-09T02:13:25,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741837_1013 (size=358) 2024-11-09T02:13:25,290 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => f28f1a8d6b5be48d6e6dca7a81da9c57, NAME => 'test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,291 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 195f23de0822a5bc7d5a1a2b4b16afdb, NAME => 'test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,291 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bb1296b406bf2a79b39a6de9a4389fcc, NAME => 'test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,299 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,299 WARN [IPC Server handler 4 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,299 WARN [IPC Server handler 4 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,299 WARN [IPC Server handler 1 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,299 WARN [IPC Server handler 4 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,299 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741840_1016 (size=45) 2024-11-09T02:13:25,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741840_1016 (size=45) 2024-11-09T02:13:25,316 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,316 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing f28f1a8d6b5be48d6e6dca7a81da9c57, disabling compactions & flushes 2024-11-09T02:13:25,317 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:25,317 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:25,317 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. after waiting 0 ms 2024-11-09T02:13:25,317 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:25,317 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:25,317 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for f28f1a8d6b5be48d6e6dca7a81da9c57: Waiting for close lock at 1731118405316Disabling compacts and flushes for region at 1731118405316Disabling writes for close at 1731118405317 (+1 ms)Writing region close event to WAL at 1731118405317Closed at 1731118405317 2024-11-09T02:13:25,318 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 5f3f9d6837573933dea12bd60465775e, NAME => 'test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741839_1015 (size=42) 2024-11-09T02:13:25,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741839_1015 (size=42) 2024-11-09T02:13:25,326 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,326 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing bb1296b406bf2a79b39a6de9a4389fcc, disabling compactions & flushes 2024-11-09T02:13:25,326 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:25,326 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:25,326 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. after waiting 0 ms 2024-11-09T02:13:25,326 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:25,326 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:25,326 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for bb1296b406bf2a79b39a6de9a4389fcc: Waiting for close lock at 1731118405326Disabling compacts and flushes for region at 1731118405326Disabling writes for close at 1731118405326Writing region close event to WAL at 1731118405326Closed at 1731118405326 2024-11-09T02:13:25,327 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5c24783f69139da02a463f7c484b941b, NAME => 'test,eee,1731118405247.5c24783f69139da02a463f7c484b941b.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,334 WARN [IPC Server handler 0 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,334 WARN [IPC Server handler 0 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,334 WARN [IPC Server handler 0 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741838_1014 (size=45) 2024-11-09T02:13:25,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741838_1014 (size=45) 2024-11-09T02:13:25,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741838_1014 (size=45) 2024-11-09T02:13:25,337 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,337 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 195f23de0822a5bc7d5a1a2b4b16afdb, disabling compactions & flushes 2024-11-09T02:13:25,337 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:25,338 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:25,338 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. after waiting 0 ms 2024-11-09T02:13:25,338 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:25,338 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:25,338 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 195f23de0822a5bc7d5a1a2b4b16afdb: Waiting for close lock at 1731118405337Disabling compacts and flushes for region at 1731118405337Disabling writes for close at 1731118405338 (+1 ms)Writing region close event to WAL at 1731118405338Closed at 1731118405338 2024-11-09T02:13:25,339 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8aa375f3cce51bfb890245ebbdc0e977, NAME => 'test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,342 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,342 WARN [IPC Server handler 1 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,342 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,350 WARN [IPC Server handler 2 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,350 WARN [IPC Server handler 2 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,350 WARN [IPC Server handler 2 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741841_1017 (size=45) 2024-11-09T02:13:25,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741841_1017 (size=45) 2024-11-09T02:13:25,353 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,353 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 5f3f9d6837573933dea12bd60465775e, disabling compactions & flushes 2024-11-09T02:13:25,353 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:25,353 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:25,354 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. after waiting 1 ms 2024-11-09T02:13:25,354 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:25,354 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:25,354 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 5f3f9d6837573933dea12bd60465775e: Waiting for close lock at 1731118405353Disabling compacts and flushes for region at 1731118405353Disabling writes for close at 1731118405354 (+1 ms)Writing region close event to WAL at 1731118405354Closed at 1731118405354 2024-11-09T02:13:25,354 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 072df2da98728631979ee233c5a94c13, NAME => 'test,ggg,1731118405247.072df2da98728631979ee233c5a94c13.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,361 WARN [IPC Server handler 0 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,362 WARN [IPC Server handler 0 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,362 WARN [IPC Server handler 0 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741842_1018 (size=45) 2024-11-09T02:13:25,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741842_1018 (size=45) 2024-11-09T02:13:25,364 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1731118405247.5c24783f69139da02a463f7c484b941b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,364 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5c24783f69139da02a463f7c484b941b, disabling compactions & flushes 2024-11-09T02:13:25,364 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:25,364 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:25,364 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. after waiting 0 ms 2024-11-09T02:13:25,364 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:25,364 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:25,365 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5c24783f69139da02a463f7c484b941b: Waiting for close lock at 1731118405364Disabling compacts and flushes for region at 1731118405364Disabling writes for close at 1731118405364Writing region close event to WAL at 1731118405364Closed at 1731118405364 2024-11-09T02:13:25,366 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6dd1285d59409a10928f32885afd0b51, NAME => 'test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741843_1019 (size=45) 2024-11-09T02:13:25,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741843_1019 (size=45) 2024-11-09T02:13:25,373 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,374 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,374 WARN [IPC Server handler 1 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,374 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,374 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8aa375f3cce51bfb890245ebbdc0e977, disabling compactions & flushes 2024-11-09T02:13:25,374 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:25,374 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:25,374 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. after waiting 0 ms 2024-11-09T02:13:25,374 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:25,374 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:25,374 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8aa375f3cce51bfb890245ebbdc0e977: Waiting for close lock at 1731118405374Disabling compacts and flushes for region at 1731118405374Disabling writes for close at 1731118405374Writing region close event to WAL at 1731118405374Closed at 1731118405374 2024-11-09T02:13:25,375 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 555bb0a1c37bd4750a7abd799c897bc7, NAME => 'test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741844_1020 (size=45) 2024-11-09T02:13:25,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741844_1020 (size=45) 2024-11-09T02:13:25,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1731118405247.072df2da98728631979ee233c5a94c13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 072df2da98728631979ee233c5a94c13, disabling compactions & flushes 2024-11-09T02:13:25,381 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:25,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:25,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. after waiting 0 ms 2024-11-09T02:13:25,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:25,381 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:25,381 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 072df2da98728631979ee233c5a94c13: Waiting for close lock at 1731118405381Disabling compacts and flushes for region at 1731118405381Disabling writes for close at 1731118405381Writing region close event to WAL at 1731118405381Closed at 1731118405381 2024-11-09T02:13:25,382 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 8fe76b2cbbde488b7620457c1f89dcc9, NAME => 'test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T02:13:25,391 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741845_1021 (size=45) 2024-11-09T02:13:25,391 WARN [IPC Server handler 3 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,391 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741845_1021 (size=45) 2024-11-09T02:13:25,393 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,393 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6dd1285d59409a10928f32885afd0b51, disabling compactions & flushes 2024-11-09T02:13:25,393 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:25,393 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:25,394 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. after waiting 0 ms 2024-11-09T02:13:25,394 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:25,394 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:25,394 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6dd1285d59409a10928f32885afd0b51: Waiting for close lock at 1731118405393Disabling compacts and flushes for region at 1731118405393Disabling writes for close at 1731118405394 (+1 ms)Writing region close event to WAL at 1731118405394Closed at 1731118405394 2024-11-09T02:13:25,395 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a5a881b27904120149acda46e3d36994, NAME => 'test,kkk,1731118405247.a5a881b27904120149acda46e3d36994.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,406 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,406 WARN [IPC Server handler 3 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,406 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741847_1023 (size=45) 2024-11-09T02:13:25,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741846_1022 (size=45) 2024-11-09T02:13:25,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741846_1022 (size=45) 2024-11-09T02:13:25,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741846_1022 (size=45) 2024-11-09T02:13:25,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741847_1023 (size=45) 2024-11-09T02:13:25,409 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,409 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,410 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 555bb0a1c37bd4750a7abd799c897bc7, disabling compactions & flushes 2024-11-09T02:13:25,410 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 8fe76b2cbbde488b7620457c1f89dcc9, disabling compactions & flushes 2024-11-09T02:13:25,410 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:25,410 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:25,410 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:25,410 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:25,410 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. after waiting 0 ms 2024-11-09T02:13:25,410 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. after waiting 0 ms 2024-11-09T02:13:25,410 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:25,410 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:25,410 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:25,410 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:25,410 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 8fe76b2cbbde488b7620457c1f89dcc9: Waiting for close lock at 1731118405410Disabling compacts and flushes for region at 1731118405410Disabling writes for close at 1731118405410Writing region close event to WAL at 1731118405410Closed at 1731118405410 2024-11-09T02:13:25,410 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 555bb0a1c37bd4750a7abd799c897bc7: Waiting for close lock at 1731118405410Disabling compacts and flushes for region at 1731118405410Disabling writes for close at 1731118405410Writing region close event to WAL at 1731118405410Closed at 1731118405410 2024-11-09T02:13:25,411 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => b3b3ba949727dd9291d8adc925fb09cb, NAME => 'test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,411 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => ad814e81d5536ce802390dd5602e78a1, NAME => 'test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,417 WARN [IPC Server handler 4 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,417 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,417 WARN [IPC Server handler 4 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,418 WARN [IPC Server handler 3 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,418 WARN [IPC Server handler 4 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,418 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741848_1024 (size=45) 2024-11-09T02:13:25,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741848_1024 (size=45) 2024-11-09T02:13:25,424 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,kkk,1731118405247.a5a881b27904120149acda46e3d36994.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,424 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing a5a881b27904120149acda46e3d36994, disabling compactions & flushes 2024-11-09T02:13:25,424 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:25,424 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:25,424 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. after waiting 0 ms 2024-11-09T02:13:25,425 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:25,425 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:25,425 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for a5a881b27904120149acda46e3d36994: Waiting for close lock at 1731118405424Disabling compacts and flushes for region at 1731118405424Disabling writes for close at 1731118405424Writing region close event to WAL at 1731118405425 (+1 ms)Closed at 1731118405425 2024-11-09T02:13:25,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741849_1025 (size=45) 2024-11-09T02:13:25,426 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 244d470b9561d8921c93364fa1596bac, NAME => 'test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741849_1025 (size=45) 2024-11-09T02:13:25,427 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,427 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing b3b3ba949727dd9291d8adc925fb09cb, disabling compactions & flushes 2024-11-09T02:13:25,427 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:25,427 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:25,427 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. after waiting 0 ms 2024-11-09T02:13:25,427 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:25,428 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:25,428 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for b3b3ba949727dd9291d8adc925fb09cb: Waiting for close lock at 1731118405427Disabling compacts and flushes for region at 1731118405427Disabling writes for close at 1731118405427Writing region close event to WAL at 1731118405427Closed at 1731118405427 2024-11-09T02:13:25,429 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 88ba7a90c8e825401b464926d88f3b36, NAME => 'test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741850_1026 (size=45) 2024-11-09T02:13:25,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741850_1026 (size=45) 2024-11-09T02:13:25,432 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,432 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing ad814e81d5536ce802390dd5602e78a1, disabling compactions & flushes 2024-11-09T02:13:25,432 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:25,432 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:25,432 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. after waiting 0 ms 2024-11-09T02:13:25,432 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:25,432 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:25,433 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for ad814e81d5536ce802390dd5602e78a1: Waiting for close lock at 1731118405432Disabling compacts and flushes for region at 1731118405432Disabling writes for close at 1731118405432Writing region close event to WAL at 1731118405432Closed at 1731118405432 2024-11-09T02:13:25,433 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 7cb62548224e02c76c614c52af2aadca, NAME => 'test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,437 WARN [IPC Server handler 0 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,437 WARN [IPC Server handler 0 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,437 WARN [IPC Server handler 0 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741851_1027 (size=45) 2024-11-09T02:13:25,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741851_1027 (size=45) 2024-11-09T02:13:25,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741851_1027 (size=45) 2024-11-09T02:13:25,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 244d470b9561d8921c93364fa1596bac, disabling compactions & flushes 2024-11-09T02:13:25,441 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:25,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:25,441 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. after waiting 0 ms 2024-11-09T02:13:25,442 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:25,442 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:25,442 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 244d470b9561d8921c93364fa1596bac: Waiting for close lock at 1731118405441Disabling compacts and flushes for region at 1731118405441Disabling writes for close at 1731118405441Writing region close event to WAL at 1731118405442 (+1 ms)Closed at 1731118405442 2024-11-09T02:13:25,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741852_1028 (size=45) 2024-11-09T02:13:25,443 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 923e0410f16fe611d9240651fc8bf39e, NAME => 'test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741852_1028 (size=45) 2024-11-09T02:13:25,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741853_1029 (size=45) 2024-11-09T02:13:25,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741853_1029 (size=45) 2024-11-09T02:13:25,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741852_1028 (size=45) 2024-11-09T02:13:25,451 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,451 WARN [IPC Server handler 4 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,451 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,451 WARN [IPC Server handler 4 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,451 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 88ba7a90c8e825401b464926d88f3b36, disabling compactions & flushes 2024-11-09T02:13:25,451 WARN [IPC Server handler 4 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,451 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:25,451 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 7cb62548224e02c76c614c52af2aadca, disabling compactions & flushes 2024-11-09T02:13:25,452 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:25,452 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:25,452 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. after waiting 0 ms 2024-11-09T02:13:25,452 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:25,452 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:25,452 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. after waiting 0 ms 2024-11-09T02:13:25,452 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:25,452 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:25,452 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:25,452 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 88ba7a90c8e825401b464926d88f3b36: Waiting for close lock at 1731118405451Disabling compacts and flushes for region at 1731118405451Disabling writes for close at 1731118405452 (+1 ms)Writing region close event to WAL at 1731118405452Closed at 1731118405452 2024-11-09T02:13:25,452 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 7cb62548224e02c76c614c52af2aadca: Waiting for close lock at 1731118405451Disabling compacts and flushes for region at 1731118405451Disabling writes for close at 1731118405452 (+1 ms)Writing region close event to WAL at 1731118405452Closed at 1731118405452 2024-11-09T02:13:25,453 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => c5c32774defefb99b01b165b020a41fa, NAME => 'test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,454 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 3a2a5d6e6a65b43660fcb665b4ef5c83, NAME => 'test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741854_1030 (size=45) 2024-11-09T02:13:25,458 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741854_1030 (size=45) 2024-11-09T02:13:25,458 WARN [IPC Server handler 1 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,459 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,459 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,460 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 923e0410f16fe611d9240651fc8bf39e, disabling compactions & flushes 2024-11-09T02:13:25,460 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:25,460 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:25,460 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. after waiting 0 ms 2024-11-09T02:13:25,460 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:25,460 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:25,460 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 923e0410f16fe611d9240651fc8bf39e: Waiting for close lock at 1731118405459Disabling compacts and flushes for region at 1731118405459Disabling writes for close at 1731118405460 (+1 ms)Writing region close event to WAL at 1731118405460Closed at 1731118405460 2024-11-09T02:13:25,461 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 194b5628d516a374d3336a52e639da04, NAME => 'test,ttt,1731118405247.194b5628d516a374d3336a52e639da04.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,466 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,466 WARN [IPC Server handler 1 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,466 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741855_1031 (size=45) 2024-11-09T02:13:25,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741855_1031 (size=45) 2024-11-09T02:13:25,468 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,468 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 3a2a5d6e6a65b43660fcb665b4ef5c83, disabling compactions & flushes 2024-11-09T02:13:25,468 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:25,468 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:25,468 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. after waiting 0 ms 2024-11-09T02:13:25,468 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:25,468 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:25,469 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 3a2a5d6e6a65b43660fcb665b4ef5c83: Waiting for close lock at 1731118405468Disabling compacts and flushes for region at 1731118405468Disabling writes for close at 1731118405468Writing region close event to WAL at 1731118405468Closed at 1731118405468 2024-11-09T02:13:25,469 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 87e2ba48090bed7cebebc5b429271688, NAME => 'test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741856_1032 (size=45) 2024-11-09T02:13:25,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741856_1032 (size=45) 2024-11-09T02:13:25,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741856_1032 (size=45) 2024-11-09T02:13:25,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing c5c32774defefb99b01b165b020a41fa, disabling compactions & flushes 2024-11-09T02:13:25,474 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:25,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:25,474 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. after waiting 0 ms 2024-11-09T02:13:25,475 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:25,475 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:25,475 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for c5c32774defefb99b01b165b020a41fa: Waiting for close lock at 1731118405474Disabling compacts and flushes for region at 1731118405474Disabling writes for close at 1731118405475 (+1 ms)Writing region close event to WAL at 1731118405475Closed at 1731118405475 2024-11-09T02:13:25,475 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => bbfdf2901c997d044e2c20f81e3a2a90, NAME => 'test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,479 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,479 WARN [IPC Server handler 1 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,480 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741857_1033 (size=45) 2024-11-09T02:13:25,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741857_1033 (size=45) 2024-11-09T02:13:25,482 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,ttt,1731118405247.194b5628d516a374d3336a52e639da04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,482 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 194b5628d516a374d3336a52e639da04, disabling compactions & flushes 2024-11-09T02:13:25,482 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:25,482 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:25,482 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. after waiting 0 ms 2024-11-09T02:13:25,482 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:25,482 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:25,482 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 194b5628d516a374d3336a52e639da04: Waiting for close lock at 1731118405482Disabling compacts and flushes for region at 1731118405482Disabling writes for close at 1731118405482Writing region close event to WAL at 1731118405482Closed at 1731118405482 2024-11-09T02:13:25,483 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f372b843955cc5a3ace82b165e5d33b0, NAME => 'test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,487 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,487 WARN [IPC Server handler 1 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,487 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741859_1035 (size=45) 2024-11-09T02:13:25,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741858_1034 (size=45) 2024-11-09T02:13:25,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741858_1034 (size=45) 2024-11-09T02:13:25,490 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,491 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing bbfdf2901c997d044e2c20f81e3a2a90, disabling compactions & flushes 2024-11-09T02:13:25,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741859_1035 (size=45) 2024-11-09T02:13:25,491 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:25,491 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:25,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,491 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. after waiting 0 ms 2024-11-09T02:13:25,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741858_1034 (size=45) 2024-11-09T02:13:25,491 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:25,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 87e2ba48090bed7cebebc5b429271688, disabling compactions & flushes 2024-11-09T02:13:25,491 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:25,491 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:25,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:25,491 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for bbfdf2901c997d044e2c20f81e3a2a90: Waiting for close lock at 1731118405491Disabling compacts and flushes for region at 1731118405491Disabling writes for close at 1731118405491Writing region close event to WAL at 1731118405491Closed at 1731118405491 2024-11-09T02:13:25,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. after waiting 0 ms 2024-11-09T02:13:25,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:25,491 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:25,491 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 87e2ba48090bed7cebebc5b429271688: Waiting for close lock at 1731118405491Disabling compacts and flushes for region at 1731118405491Disabling writes for close at 1731118405491Writing region close event to WAL at 1731118405491Closed at 1731118405491 2024-11-09T02:13:25,492 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => e33ccfa49d4634ee727e7cd03b929a72, NAME => 'test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,492 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 85da2d651446630bfba59fcb89f923a1, NAME => 'test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,497 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,497 WARN [IPC Server handler 1 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,497 WARN [IPC Server handler 1 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741860_1036 (size=45) 2024-11-09T02:13:25,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741860_1036 (size=45) 2024-11-09T02:13:25,501 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,502 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing f372b843955cc5a3ace82b165e5d33b0, disabling compactions & flushes 2024-11-09T02:13:25,502 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:25,502 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:25,502 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. after waiting 0 ms 2024-11-09T02:13:25,502 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:25,502 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:25,502 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for f372b843955cc5a3ace82b165e5d33b0: Waiting for close lock at 1731118405502Disabling compacts and flushes for region at 1731118405502Disabling writes for close at 1731118405502Writing region close event to WAL at 1731118405502Closed at 1731118405502 2024-11-09T02:13:25,503 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6e77447186c114b3577a6f77b1ffe798, NAME => 'test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8 2024-11-09T02:13:25,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741862_1038 (size=45) 2024-11-09T02:13:25,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741862_1038 (size=45) 2024-11-09T02:13:25,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741861_1037 (size=45) 2024-11-09T02:13:25,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741861_1037 (size=45) 2024-11-09T02:13:25,507 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,507 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 85da2d651446630bfba59fcb89f923a1, disabling compactions & flushes 2024-11-09T02:13:25,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741861_1037 (size=45) 2024-11-09T02:13:25,507 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:25,507 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:25,507 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. after waiting 0 ms 2024-11-09T02:13:25,508 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:25,508 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:25,508 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 85da2d651446630bfba59fcb89f923a1: Waiting for close lock at 1731118405507Disabling compacts and flushes for region at 1731118405507Disabling writes for close at 1731118405507Writing region close event to WAL at 1731118405508 (+1 ms)Closed at 1731118405508 2024-11-09T02:13:25,508 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,508 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing e33ccfa49d4634ee727e7cd03b929a72, disabling compactions & flushes 2024-11-09T02:13:25,508 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:25,509 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:25,509 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. after waiting 0 ms 2024-11-09T02:13:25,509 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:25,509 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:25,509 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for e33ccfa49d4634ee727e7cd03b929a72: Waiting for close lock at 1731118405508Disabling compacts and flushes for region at 1731118405508Disabling writes for close at 1731118405509 (+1 ms)Writing region close event to WAL at 1731118405509Closed at 1731118405509 2024-11-09T02:13:25,509 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-09T02:13:25,509 WARN [IPC Server handler 3 on default port 45553 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-09T02:13:25,509 WARN [IPC Server handler 3 on default port 45553 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-09T02:13:25,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741863_1039 (size=42) 2024-11-09T02:13:25,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741863_1039 (size=42) 2024-11-09T02:13:25,514 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,514 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6e77447186c114b3577a6f77b1ffe798, disabling compactions & flushes 2024-11-09T02:13:25,515 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:25,515 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:25,515 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. after waiting 0 ms 2024-11-09T02:13:25,515 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:25,515 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:25,515 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6e77447186c114b3577a6f77b1ffe798: Waiting for close lock at 1731118405514Disabling compacts and flushes for region at 1731118405514Disabling writes for close at 1731118405515 (+1 ms)Writing region close event to WAL at 1731118405515Closed at 1731118405515 2024-11-09T02:13:25,517 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-09T02:13:25,523 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,523 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,523 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,523 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,523 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1731118405247.5c24783f69139da02a463f7c484b941b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,523 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,524 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1731118405247.072df2da98728631979ee233c5a94c13.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,524 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,524 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,524 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,524 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1731118405247.a5a881b27904120149acda46e3d36994.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,525 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,525 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,525 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,525 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,525 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,525 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,525 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1731118405247.194b5628d516a374d3336a52e639da04.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,526 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731118405518"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731118405518"}]},"ts":"1731118405518"} 2024-11-09T02:13:25,567 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-09T02:13:25,569 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-09T02:13:25,571 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118405569"}]},"ts":"1731118405569"} 2024-11-09T02:13:25,575 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-09T02:13:25,576 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {53f286d3af3b=0} racks are {/default-rack=0} 2024-11-09T02:13:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T02:13:25,594 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T02:13:25,594 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T02:13:25,594 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-09T02:13:25,594 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T02:13:25,594 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T02:13:25,594 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-09T02:13:25,594 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T02:13:25,594 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T02:13:25,594 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-09T02:13:25,595 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T02:13:25,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb1296b406bf2a79b39a6de9a4389fcc, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f28f1a8d6b5be48d6e6dca7a81da9c57, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5f3f9d6837573933dea12bd60465775e, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5c24783f69139da02a463f7c484b941b, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=072df2da98728631979ee233c5a94c13, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6dd1285d59409a10928f32885afd0b51, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=555bb0a1c37bd4750a7abd799c897bc7, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a5a881b27904120149acda46e3d36994, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad814e81d5536ce802390dd5602e78a1, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=244d470b9561d8921c93364fa1596bac, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7cb62548224e02c76c614c52af2aadca, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=923e0410f16fe611d9240651fc8bf39e, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c5c32774defefb99b01b165b020a41fa, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=194b5628d516a374d3336a52e639da04, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87e2ba48090bed7cebebc5b429271688, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f372b843955cc5a3ace82b165e5d33b0, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e33ccfa49d4634ee727e7cd03b929a72, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e77447186c114b3577a6f77b1ffe798, ASSIGN}] 2024-11-09T02:13:25,608 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6e77447186c114b3577a6f77b1ffe798, ASSIGN 2024-11-09T02:13:25,610 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6e77447186c114b3577a6f77b1ffe798, ASSIGN; state=OFFLINE, location=53f286d3af3b,45941,1731118402318; forceNewPlan=false, retain=false 2024-11-09T02:13:25,611 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:25,612 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN; state=OFFLINE, location=53f286d3af3b,33749,1731118402489; forceNewPlan=false, retain=false 2024-11-09T02:13:25,614 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=e33ccfa49d4634ee727e7cd03b929a72, ASSIGN 2024-11-09T02:13:25,615 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=e33ccfa49d4634ee727e7cd03b929a72, ASSIGN; state=OFFLINE, location=53f286d3af3b,46883,1731118402441; forceNewPlan=false, retain=false 2024-11-09T02:13:25,617 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f372b843955cc5a3ace82b165e5d33b0, ASSIGN 2024-11-09T02:13:25,618 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f372b843955cc5a3ace82b165e5d33b0, ASSIGN; state=OFFLINE, location=53f286d3af3b,45941,1731118402318; forceNewPlan=false, retain=false 2024-11-09T02:13:25,619 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:25,621 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN; state=OFFLINE, location=53f286d3af3b,33749,1731118402489; forceNewPlan=false, retain=false 2024-11-09T02:13:25,623 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=87e2ba48090bed7cebebc5b429271688, ASSIGN 2024-11-09T02:13:25,624 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=87e2ba48090bed7cebebc5b429271688, ASSIGN; state=OFFLINE, location=53f286d3af3b,46883,1731118402441; forceNewPlan=false, retain=false 2024-11-09T02:13:25,626 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=194b5628d516a374d3336a52e639da04, ASSIGN 2024-11-09T02:13:25,627 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=194b5628d516a374d3336a52e639da04, ASSIGN; state=OFFLINE, location=53f286d3af3b,45941,1731118402318; forceNewPlan=false, retain=false 2024-11-09T02:13:25,629 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c5c32774defefb99b01b165b020a41fa, ASSIGN 2024-11-09T02:13:25,630 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c5c32774defefb99b01b165b020a41fa, ASSIGN; state=OFFLINE, location=53f286d3af3b,46883,1731118402441; forceNewPlan=false, retain=false 2024-11-09T02:13:25,632 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:25,633 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN; state=OFFLINE, location=53f286d3af3b,33749,1731118402489; forceNewPlan=false, retain=false 2024-11-09T02:13:25,634 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=923e0410f16fe611d9240651fc8bf39e, ASSIGN 2024-11-09T02:13:25,636 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=923e0410f16fe611d9240651fc8bf39e, ASSIGN; state=OFFLINE, location=53f286d3af3b,45941,1731118402318; forceNewPlan=false, retain=false 2024-11-09T02:13:25,637 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=7cb62548224e02c76c614c52af2aadca, ASSIGN 2024-11-09T02:13:25,638 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=7cb62548224e02c76c614c52af2aadca, ASSIGN; state=OFFLINE, location=53f286d3af3b,46883,1731118402441; forceNewPlan=false, retain=false 2024-11-09T02:13:25,640 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:25,641 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN; state=OFFLINE, location=53f286d3af3b,33749,1731118402489; forceNewPlan=false, retain=false 2024-11-09T02:13:25,643 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=244d470b9561d8921c93364fa1596bac, ASSIGN 2024-11-09T02:13:25,644 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=244d470b9561d8921c93364fa1596bac, ASSIGN; state=OFFLINE, location=53f286d3af3b,45941,1731118402318; forceNewPlan=false, retain=false 2024-11-09T02:13:25,646 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:25,647 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN; state=OFFLINE, location=53f286d3af3b,33749,1731118402489; forceNewPlan=false, retain=false 2024-11-09T02:13:25,649 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ad814e81d5536ce802390dd5602e78a1, ASSIGN 2024-11-09T02:13:25,650 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ad814e81d5536ce802390dd5602e78a1, ASSIGN; state=OFFLINE, location=53f286d3af3b,46883,1731118402441; forceNewPlan=false, retain=false 2024-11-09T02:13:25,651 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=a5a881b27904120149acda46e3d36994, ASSIGN 2024-11-09T02:13:25,653 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=a5a881b27904120149acda46e3d36994, ASSIGN; state=OFFLINE, location=53f286d3af3b,45941,1731118402318; forceNewPlan=false, retain=false 2024-11-09T02:13:25,654 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:25,655 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN; state=OFFLINE, location=53f286d3af3b,33749,1731118402489; forceNewPlan=false, retain=false 2024-11-09T02:13:25,658 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=555bb0a1c37bd4750a7abd799c897bc7, ASSIGN 2024-11-09T02:13:25,659 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=555bb0a1c37bd4750a7abd799c897bc7, ASSIGN; state=OFFLINE, location=53f286d3af3b,46883,1731118402441; forceNewPlan=false, retain=false 2024-11-09T02:13:25,661 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6dd1285d59409a10928f32885afd0b51, ASSIGN 2024-11-09T02:13:25,662 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6dd1285d59409a10928f32885afd0b51, ASSIGN; state=OFFLINE, location=53f286d3af3b,45941,1731118402318; forceNewPlan=false, retain=false 2024-11-09T02:13:25,663 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=072df2da98728631979ee233c5a94c13, ASSIGN 2024-11-09T02:13:25,665 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=072df2da98728631979ee233c5a94c13, ASSIGN; state=OFFLINE, location=53f286d3af3b,46883,1731118402441; forceNewPlan=false, retain=false 2024-11-09T02:13:25,666 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:25,667 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN; state=OFFLINE, location=53f286d3af3b,33749,1731118402489; forceNewPlan=false, retain=false 2024-11-09T02:13:25,668 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5c24783f69139da02a463f7c484b941b, ASSIGN 2024-11-09T02:13:25,669 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5c24783f69139da02a463f7c484b941b, ASSIGN; state=OFFLINE, location=53f286d3af3b,45941,1731118402318; forceNewPlan=false, retain=false 2024-11-09T02:13:25,671 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5f3f9d6837573933dea12bd60465775e, ASSIGN 2024-11-09T02:13:25,672 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5f3f9d6837573933dea12bd60465775e, ASSIGN; state=OFFLINE, location=53f286d3af3b,46883,1731118402441; forceNewPlan=false, retain=false 2024-11-09T02:13:25,674 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f28f1a8d6b5be48d6e6dca7a81da9c57, ASSIGN 2024-11-09T02:13:25,675 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f28f1a8d6b5be48d6e6dca7a81da9c57, ASSIGN; state=OFFLINE, location=53f286d3af3b,46883,1731118402441; forceNewPlan=false, retain=false 2024-11-09T02:13:25,677 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:25,678 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN; state=OFFLINE, location=53f286d3af3b,33749,1731118402489; forceNewPlan=false, retain=false 2024-11-09T02:13:25,681 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bb1296b406bf2a79b39a6de9a4389fcc, ASSIGN 2024-11-09T02:13:25,682 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bb1296b406bf2a79b39a6de9a4389fcc, ASSIGN; state=OFFLINE, location=53f286d3af3b,45941,1731118402318; forceNewPlan=false, retain=false 2024-11-09T02:13:25,768 INFO [53f286d3af3b:36011 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-09T02:13:25,769 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:25,770 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=87e2ba48090bed7cebebc5b429271688, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:25,771 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c5c32774defefb99b01b165b020a41fa, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:25,772 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7cb62548224e02c76c614c52af2aadca, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:25,773 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ad814e81d5536ce802390dd5602e78a1, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:25,774 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:25,774 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=072df2da98728631979ee233c5a94c13, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:25,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=e33ccfa49d4634ee727e7cd03b929a72, ASSIGN because future has completed 2024-11-09T02:13:25,775 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5f3f9d6837573933dea12bd60465775e, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:25,776 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure e33ccfa49d4634ee727e7cd03b929a72, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:25,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=87e2ba48090bed7cebebc5b429271688, ASSIGN because future has completed 2024-11-09T02:13:25,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c5c32774defefb99b01b165b020a41fa, ASSIGN because future has completed 2024-11-09T02:13:25,783 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=7cb62548224e02c76c614c52af2aadca, ASSIGN because future has completed 2024-11-09T02:13:25,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5c32774defefb99b01b165b020a41fa, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:25,786 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ad814e81d5536ce802390dd5602e78a1, ASSIGN because future has completed 2024-11-09T02:13:25,787 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=555bb0a1c37bd4750a7abd799c897bc7, ASSIGN because future has completed 2024-11-09T02:13:25,790 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=072df2da98728631979ee233c5a94c13, ASSIGN because future has completed 2024-11-09T02:13:25,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 555bb0a1c37bd4750a7abd799c897bc7, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:25,793 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5f3f9d6837573933dea12bd60465775e, ASSIGN because future has completed 2024-11-09T02:13:25,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f3f9d6837573933dea12bd60465775e, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:25,803 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 072df2da98728631979ee233c5a94c13, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:25,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad814e81d5536ce802390dd5602e78a1, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:25,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cb62548224e02c76c614c52af2aadca, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:25,820 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87e2ba48090bed7cebebc5b429271688, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:25,826 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:25,826 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=6e77447186c114b3577a6f77b1ffe798, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:25,827 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f372b843955cc5a3ace82b165e5d33b0, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:25,827 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=194b5628d516a374d3336a52e639da04, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:25,828 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=923e0410f16fe611d9240651fc8bf39e, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:25,829 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=244d470b9561d8921c93364fa1596bac, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:25,829 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f28f1a8d6b5be48d6e6dca7a81da9c57, ASSIGN because future has completed 2024-11-09T02:13:25,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f28f1a8d6b5be48d6e6dca7a81da9c57, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:25,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6e77447186c114b3577a6f77b1ffe798, ASSIGN because future has completed 2024-11-09T02:13:25,832 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f372b843955cc5a3ace82b165e5d33b0, ASSIGN because future has completed 2024-11-09T02:13:25,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=194b5628d516a374d3336a52e639da04, ASSIGN because future has completed 2024-11-09T02:13:25,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=923e0410f16fe611d9240651fc8bf39e, ASSIGN because future has completed 2024-11-09T02:13:25,837 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923e0410f16fe611d9240651fc8bf39e, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:25,837 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=244d470b9561d8921c93364fa1596bac, ASSIGN because future has completed 2024-11-09T02:13:25,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 244d470b9561d8921c93364fa1596bac, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:25,847 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 194b5628d516a374d3336a52e639da04, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:25,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure f372b843955cc5a3ace82b165e5d33b0, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:25,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e77447186c114b3577a6f77b1ffe798, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:25,864 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a5a881b27904120149acda46e3d36994, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:25,865 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=6dd1285d59409a10928f32885afd0b51, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:25,866 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5c24783f69139da02a463f7c484b941b, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:25,867 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:25,867 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:25,868 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=a5a881b27904120149acda46e3d36994, ASSIGN because future has completed 2024-11-09T02:13:25,868 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:25,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5a881b27904120149acda46e3d36994, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:25,872 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6dd1285d59409a10928f32885afd0b51, ASSIGN because future has completed 2024-11-09T02:13:25,874 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5c24783f69139da02a463f7c484b941b, ASSIGN because future has completed 2024-11-09T02:13:25,876 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bb1296b406bf2a79b39a6de9a4389fcc, ASSIGN because future has completed 2024-11-09T02:13:25,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN because future has completed 2024-11-09T02:13:25,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,33749,1731118402489}] 2024-11-09T02:13:25,879 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN because future has completed 2024-11-09T02:13:25,884 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,33749,1731118402489}] 2024-11-09T02:13:25,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb1296b406bf2a79b39a6de9a4389fcc, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:25,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c24783f69139da02a463f7c484b941b, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:25,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dd1285d59409a10928f32885afd0b51, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:25,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T02:13:25,904 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:25,905 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:25,906 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:25,906 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:25,907 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:25,907 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:25,908 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN because future has completed 2024-11-09T02:13:25,908 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a2a5d6e6a65b43660fcb665b4ef5c83, server=53f286d3af3b,33749,1731118402489}] 2024-11-09T02:13:25,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN because future has completed 2024-11-09T02:13:25,911 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN because future has completed 2024-11-09T02:13:25,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN because future has completed 2024-11-09T02:13:25,912 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,33749,1731118402489}] 2024-11-09T02:13:25,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN because future has completed 2024-11-09T02:13:25,915 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN because future has completed 2024-11-09T02:13:25,917 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,33749,1731118402489}] 2024-11-09T02:13:25,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,33749,1731118402489}] 2024-11-09T02:13:25,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,33749,1731118402489}] 2024-11-09T02:13:25,931 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T02:13:25,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39629, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T02:13:25,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,33749,1731118402489}] 2024-11-09T02:13:25,942 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:25,943 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => f28f1a8d6b5be48d6e6dca7a81da9c57, NAME => 'test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-09T02:13:25,943 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:25,943 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,943 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:25,943 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:25,948 INFO [StoreOpener-f28f1a8d6b5be48d6e6dca7a81da9c57-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:25,950 INFO [StoreOpener-f28f1a8d6b5be48d6e6dca7a81da9c57-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f28f1a8d6b5be48d6e6dca7a81da9c57 columnFamilyName family 2024-11-09T02:13:25,950 DEBUG [StoreOpener-f28f1a8d6b5be48d6e6dca7a81da9c57-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:25,951 INFO [StoreOpener-f28f1a8d6b5be48d6e6dca7a81da9c57-1 {}] regionserver.HStore(327): Store=f28f1a8d6b5be48d6e6dca7a81da9c57/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:25,952 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:25,953 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:25,953 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:25,954 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:25,954 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:25,956 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:25,960 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/f28f1a8d6b5be48d6e6dca7a81da9c57/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:25,960 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened f28f1a8d6b5be48d6e6dca7a81da9c57; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59882800, jitterRate=-0.10767674446105957}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:25,961 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:25,961 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for f28f1a8d6b5be48d6e6dca7a81da9c57: Running coprocessor pre-open hook at 1731118405944Writing region info on filesystem at 1731118405944Initializing all the Stores at 1731118405947 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118405947Cleaning up temporary data from old regions at 1731118405954 (+7 ms)Running coprocessor post-open hooks at 1731118405961 (+7 ms)Region opened successfully at 1731118405961 2024-11-09T02:13:25,963 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57., pid=39, masterSystemTime=1731118405930 2024-11-09T02:13:25,966 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:25,966 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:25,967 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:25,967 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => c5c32774defefb99b01b165b020a41fa, NAME => 'test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-09T02:13:25,967 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:25,968 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,968 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:25,968 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:25,968 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:25,971 INFO [StoreOpener-c5c32774defefb99b01b165b020a41fa-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:25,971 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure f28f1a8d6b5be48d6e6dca7a81da9c57, server=53f286d3af3b,46883,1731118402441 because future has completed 2024-11-09T02:13:25,973 INFO [StoreOpener-c5c32774defefb99b01b165b020a41fa-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c5c32774defefb99b01b165b020a41fa columnFamilyName family 2024-11-09T02:13:25,973 DEBUG [StoreOpener-c5c32774defefb99b01b165b020a41fa-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:25,975 INFO [StoreOpener-c5c32774defefb99b01b165b020a41fa-1 {}] regionserver.HStore(327): Store=c5c32774defefb99b01b165b020a41fa/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:25,975 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:25,977 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:25,978 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:25,978 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:25,978 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:25,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=7 2024-11-09T02:13:25,978 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure f28f1a8d6b5be48d6e6dca7a81da9c57, server=53f286d3af3b,46883,1731118402441 in 144 msec 2024-11-09T02:13:25,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f28f1a8d6b5be48d6e6dca7a81da9c57, ASSIGN in 382 msec 2024-11-09T02:13:25,982 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:25,985 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/c5c32774defefb99b01b165b020a41fa/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:25,986 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened c5c32774defefb99b01b165b020a41fa; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75028786, jitterRate=0.11801603436470032}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:25,986 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:25,986 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for c5c32774defefb99b01b165b020a41fa: Running coprocessor pre-open hook at 1731118405968Writing region info on filesystem at 1731118405968Initializing all the Stores at 1731118405970 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118405970Cleaning up temporary data from old regions at 1731118405978 (+8 ms)Running coprocessor post-open hooks at 1731118405986 (+8 ms)Region opened successfully at 1731118405986 2024-11-09T02:13:25,988 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa., pid=32, masterSystemTime=1731118405930 2024-11-09T02:13:25,991 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:25,991 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:25,991 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-09T02:13:25,991 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:25,991 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 7cb62548224e02c76c614c52af2aadca, NAME => 'test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-09T02:13:25,992 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:25,992 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:25,992 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:25,992 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:25,993 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:25,993 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33907, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-09T02:13:25,996 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure c5c32774defefb99b01b165b020a41fa, server=53f286d3af3b,46883,1731118402441 because future has completed 2024-11-09T02:13:25,997 INFO [StoreOpener-7cb62548224e02c76c614c52af2aadca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:25,999 INFO [StoreOpener-7cb62548224e02c76c614c52af2aadca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7cb62548224e02c76c614c52af2aadca columnFamilyName family 2024-11-09T02:13:26,000 DEBUG [StoreOpener-7cb62548224e02c76c614c52af2aadca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,000 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:26,000 INFO [StoreOpener-7cb62548224e02c76c614c52af2aadca-1 {}] regionserver.HStore(327): Store=7cb62548224e02c76c614c52af2aadca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,001 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => a5a881b27904120149acda46e3d36994, NAME => 'test,kkk,1731118405247.a5a881b27904120149acda46e3d36994.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-09T02:13:26,001 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:26,001 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test a5a881b27904120149acda46e3d36994 2024-11-09T02:13:26,001 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,kkk,1731118405247.a5a881b27904120149acda46e3d36994.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,001 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for a5a881b27904120149acda46e3d36994 2024-11-09T02:13:26,001 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for a5a881b27904120149acda46e3d36994 2024-11-09T02:13:26,002 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:26,003 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:26,004 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:26,004 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:26,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=23 2024-11-09T02:13:26,004 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure c5c32774defefb99b01b165b020a41fa, server=53f286d3af3b,46883,1731118402441 in 214 msec 2024-11-09T02:13:26,004 INFO [StoreOpener-a5a881b27904120149acda46e3d36994-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region a5a881b27904120149acda46e3d36994 2024-11-09T02:13:26,007 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:26,008 INFO [StoreOpener-a5a881b27904120149acda46e3d36994-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a5a881b27904120149acda46e3d36994 columnFamilyName family 2024-11-09T02:13:26,008 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c5c32774defefb99b01b165b020a41fa, ASSIGN in 407 msec 2024-11-09T02:13:26,008 DEBUG [StoreOpener-a5a881b27904120149acda46e3d36994-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,009 INFO [StoreOpener-a5a881b27904120149acda46e3d36994-1 {}] regionserver.HStore(327): Store=a5a881b27904120149acda46e3d36994/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,009 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for a5a881b27904120149acda46e3d36994 2024-11-09T02:13:26,010 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/a5a881b27904120149acda46e3d36994 2024-11-09T02:13:26,011 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/a5a881b27904120149acda46e3d36994 2024-11-09T02:13:26,012 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for a5a881b27904120149acda46e3d36994 2024-11-09T02:13:26,012 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for a5a881b27904120149acda46e3d36994 2024-11-09T02:13:26,013 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/7cb62548224e02c76c614c52af2aadca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,014 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 7cb62548224e02c76c614c52af2aadca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72476627, jitterRate=0.07998590171337128}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,014 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:26,014 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 7cb62548224e02c76c614c52af2aadca: Running coprocessor pre-open hook at 1731118405992Writing region info on filesystem at 1731118405992Initializing all the Stores at 1731118405994 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118405994Cleaning up temporary data from old regions at 1731118406004 (+10 ms)Running coprocessor post-open hooks at 1731118406014 (+10 ms)Region opened successfully at 1731118406014 2024-11-09T02:13:26,014 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for a5a881b27904120149acda46e3d36994 2024-11-09T02:13:26,015 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca., pid=37, masterSystemTime=1731118405930 2024-11-09T02:13:26,018 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/a5a881b27904120149acda46e3d36994/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,019 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:26,019 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:26,019 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened a5a881b27904120149acda46e3d36994; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59188795, jitterRate=-0.11801822483539581}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,019 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a5a881b27904120149acda46e3d36994 2024-11-09T02:13:26,019 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:26,019 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for a5a881b27904120149acda46e3d36994: Running coprocessor pre-open hook at 1731118406002Writing region info on filesystem at 1731118406002Initializing all the Stores at 1731118406003 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406004 (+1 ms)Cleaning up temporary data from old regions at 1731118406012 (+8 ms)Running coprocessor post-open hooks at 1731118406019 (+7 ms)Region opened successfully at 1731118406019 2024-11-09T02:13:26,020 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 87e2ba48090bed7cebebc5b429271688, NAME => 'test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-09T02:13:26,020 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:26,020 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,021 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:26,021 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:26,021 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1731118405247.a5a881b27904120149acda46e3d36994., pid=45, masterSystemTime=1731118405990 2024-11-09T02:13:26,021 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:26,024 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:26,025 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:26,025 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:26,025 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7cb62548224e02c76c614c52af2aadca, server=53f286d3af3b,46883,1731118402441 because future has completed 2024-11-09T02:13:26,025 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => 6dd1285d59409a10928f32885afd0b51, NAME => 'test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-09T02:13:26,026 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:26,026 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=a5a881b27904120149acda46e3d36994, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:26,026 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,026 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for 6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:26,026 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for 6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:26,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure a5a881b27904120149acda46e3d36994, server=53f286d3af3b,45941,1731118402318 because future has completed 2024-11-09T02:13:26,030 INFO [StoreOpener-87e2ba48090bed7cebebc5b429271688-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:26,031 INFO [StoreOpener-6dd1285d59409a10928f32885afd0b51-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:26,033 INFO [StoreOpener-6dd1285d59409a10928f32885afd0b51-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6dd1285d59409a10928f32885afd0b51 columnFamilyName family 2024-11-09T02:13:26,033 DEBUG [StoreOpener-6dd1285d59409a10928f32885afd0b51-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,034 INFO [StoreOpener-87e2ba48090bed7cebebc5b429271688-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 87e2ba48090bed7cebebc5b429271688 columnFamilyName family 2024-11-09T02:13:26,034 DEBUG [StoreOpener-87e2ba48090bed7cebebc5b429271688-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=20 2024-11-09T02:13:26,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 7cb62548224e02c76c614c52af2aadca, server=53f286d3af3b,46883,1731118402441 in 214 msec 2024-11-09T02:13:26,034 INFO [StoreOpener-6dd1285d59409a10928f32885afd0b51-1 {}] regionserver.HStore(327): Store=6dd1285d59409a10928f32885afd0b51/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,035 INFO [StoreOpener-87e2ba48090bed7cebebc5b429271688-1 {}] regionserver.HStore(327): Store=87e2ba48090bed7cebebc5b429271688/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,035 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for 6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:26,035 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:26,036 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:26,038 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:26,038 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:26,038 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=7cb62548224e02c76c614c52af2aadca, ASSIGN in 437 msec 2024-11-09T02:13:26,038 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:26,039 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 8aa375f3cce51bfb890245ebbdc0e977, NAME => 'test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-09T02:13:26,039 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:26,039 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,039 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:26,039 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for 6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:26,039 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:26,039 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for 6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:26,039 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:26,040 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:26,040 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:26,043 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for 6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:26,043 INFO [StoreOpener-8aa375f3cce51bfb890245ebbdc0e977-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:26,044 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:26,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=15 2024-11-09T02:13:26,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure a5a881b27904120149acda46e3d36994, server=53f286d3af3b,45941,1731118402318 in 171 msec 2024-11-09T02:13:26,046 INFO [StoreOpener-8aa375f3cce51bfb890245ebbdc0e977-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8aa375f3cce51bfb890245ebbdc0e977 columnFamilyName family 2024-11-09T02:13:26,046 DEBUG [StoreOpener-8aa375f3cce51bfb890245ebbdc0e977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,048 INFO [StoreOpener-8aa375f3cce51bfb890245ebbdc0e977-1 {}] regionserver.HStore(327): Store=8aa375f3cce51bfb890245ebbdc0e977/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,048 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:26,049 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/6dd1285d59409a10928f32885afd0b51/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,049 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/87e2ba48090bed7cebebc5b429271688/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,050 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=a5a881b27904120149acda46e3d36994, ASSIGN in 448 msec 2024-11-09T02:13:26,050 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:26,050 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 87e2ba48090bed7cebebc5b429271688; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70352886, jitterRate=0.04833969473838806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,050 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened 6dd1285d59409a10928f32885afd0b51; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75203894, jitterRate=0.12062534689903259}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,050 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:26,050 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:26,050 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 87e2ba48090bed7cebebc5b429271688: Running coprocessor pre-open hook at 1731118406021Writing region info on filesystem at 1731118406021Initializing all the Stores at 1731118406024 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406024Cleaning up temporary data from old regions at 1731118406041 (+17 ms)Running coprocessor post-open hooks at 1731118406050 (+9 ms)Region opened successfully at 1731118406050 2024-11-09T02:13:26,050 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for 6dd1285d59409a10928f32885afd0b51: Running coprocessor pre-open hook at 1731118406027Writing region info on filesystem at 1731118406027Initializing all the Stores at 1731118406030 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406030Cleaning up temporary data from old regions at 1731118406039 (+9 ms)Running coprocessor post-open hooks at 1731118406050 (+11 ms)Region opened successfully at 1731118406050 2024-11-09T02:13:26,051 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:26,052 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688., pid=38, masterSystemTime=1731118405930 2024-11-09T02:13:26,052 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51., pid=50, masterSystemTime=1731118405990 2024-11-09T02:13:26,052 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:26,052 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:26,055 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:26,055 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:26,055 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:26,055 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 244d470b9561d8921c93364fa1596bac, NAME => 'test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-09T02:13:26,056 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:26,056 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:26,056 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,056 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:26,056 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:26,056 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:26,057 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:26,057 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:26,057 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:26,057 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 072df2da98728631979ee233c5a94c13, NAME => 'test,ggg,1731118405247.072df2da98728631979ee233c5a94c13.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-09T02:13:26,058 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 072df2da98728631979ee233c5a94c13 2024-11-09T02:13:26,058 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,ggg,1731118405247.072df2da98728631979ee233c5a94c13.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,058 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 072df2da98728631979ee233c5a94c13 2024-11-09T02:13:26,058 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 072df2da98728631979ee233c5a94c13 2024-11-09T02:13:26,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6dd1285d59409a10928f32885afd0b51, server=53f286d3af3b,45941,1731118402318 because future has completed 2024-11-09T02:13:26,060 INFO [StoreOpener-244d470b9561d8921c93364fa1596bac-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:26,061 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:26,063 INFO [StoreOpener-244d470b9561d8921c93364fa1596bac-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 244d470b9561d8921c93364fa1596bac columnFamilyName family 2024-11-09T02:13:26,063 DEBUG [StoreOpener-244d470b9561d8921c93364fa1596bac-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,063 INFO [StoreOpener-244d470b9561d8921c93364fa1596bac-1 {}] regionserver.HStore(327): Store=244d470b9561d8921c93364fa1596bac/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,064 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:26,064 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8aa375f3cce51bfb890245ebbdc0e977/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,065 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 87e2ba48090bed7cebebc5b429271688, server=53f286d3af3b,46883,1731118402441 because future has completed 2024-11-09T02:13:26,065 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:26,065 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 8aa375f3cce51bfb890245ebbdc0e977; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63057899, jitterRate=-0.06036408245563507}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,065 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:26,066 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 8aa375f3cce51bfb890245ebbdc0e977: Running coprocessor pre-open hook at 1731118406039Writing region info on filesystem at 1731118406039Initializing all the Stores at 1731118406042 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406042Cleaning up temporary data from old regions at 1731118406052 (+10 ms)Running coprocessor post-open hooks at 1731118406065 (+13 ms)Region opened successfully at 1731118406065 2024-11-09T02:13:26,066 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:26,067 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:26,067 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:26,067 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977., pid=54, masterSystemTime=1731118406032 2024-11-09T02:13:26,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=12 2024-11-09T02:13:26,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 6dd1285d59409a10928f32885afd0b51, server=53f286d3af3b,45941,1731118402318 in 165 msec 2024-11-09T02:13:26,069 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:26,070 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:26,070 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:26,071 INFO [StoreOpener-072df2da98728631979ee233c5a94c13-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 072df2da98728631979ee233c5a94c13 2024-11-09T02:13:26,071 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:26,072 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 88ba7a90c8e825401b464926d88f3b36, NAME => 'test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-09T02:13:26,072 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:26,072 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,072 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:26,073 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6dd1285d59409a10928f32885afd0b51, ASSIGN in 472 msec 2024-11-09T02:13:26,073 INFO [StoreOpener-072df2da98728631979ee233c5a94c13-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 072df2da98728631979ee233c5a94c13 columnFamilyName family 2024-11-09T02:13:26,073 DEBUG [StoreOpener-072df2da98728631979ee233c5a94c13-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,074 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:26,074 INFO [StoreOpener-072df2da98728631979ee233c5a94c13-1 {}] regionserver.HStore(327): Store=072df2da98728631979ee233c5a94c13/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,074 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/244d470b9561d8921c93364fa1596bac/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,074 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 072df2da98728631979ee233c5a94c13 2024-11-09T02:13:26,075 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:26,075 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/072df2da98728631979ee233c5a94c13 2024-11-09T02:13:26,077 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/072df2da98728631979ee233c5a94c13 2024-11-09T02:13:26,077 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 244d470b9561d8921c93364fa1596bac; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61856607, jitterRate=-0.07826472818851471}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,077 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:26,077 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 244d470b9561d8921c93364fa1596bac: Running coprocessor pre-open hook at 1731118406056Writing region info on filesystem at 1731118406056Initializing all the Stores at 1731118406058 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406058Cleaning up temporary data from old regions at 1731118406067 (+9 ms)Running coprocessor post-open hooks at 1731118406077 (+10 ms)Region opened successfully at 1731118406077 2024-11-09T02:13:26,077 INFO [StoreOpener-88ba7a90c8e825401b464926d88f3b36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:26,078 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 072df2da98728631979ee233c5a94c13 2024-11-09T02:13:26,078 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 072df2da98728631979ee233c5a94c13 2024-11-09T02:13:26,080 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,46883,1731118402441, table=test, region=87e2ba48090bed7cebebc5b429271688. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,081 INFO [StoreOpener-88ba7a90c8e825401b464926d88f3b36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88ba7a90c8e825401b464926d88f3b36 columnFamilyName family 2024-11-09T02:13:26,081 DEBUG [StoreOpener-88ba7a90c8e825401b464926d88f3b36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,081 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,33749,1731118402489 because future has completed 2024-11-09T02:13:26,082 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac., pid=41, masterSystemTime=1731118405990 2024-11-09T02:13:26,082 INFO [StoreOpener-88ba7a90c8e825401b464926d88f3b36-1 {}] regionserver.HStore(327): Store=88ba7a90c8e825401b464926d88f3b36/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,082 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:26,083 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 072df2da98728631979ee233c5a94c13 2024-11-09T02:13:26,083 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:26,085 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:26,085 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:26,086 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:26,086 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:26,087 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:26,087 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:26,087 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => bb1296b406bf2a79b39a6de9a4389fcc, NAME => 'test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-09T02:13:26,087 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,33749,1731118402489, table=test, region=8aa375f3cce51bfb890245ebbdc0e977. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,088 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:26,088 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/072df2da98728631979ee233c5a94c13/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=25 2024-11-09T02:13:26,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 87e2ba48090bed7cebebc5b429271688, server=53f286d3af3b,46883,1731118402441 in 262 msec 2024-11-09T02:13:26,089 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,089 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 072df2da98728631979ee233c5a94c13; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68998469, jitterRate=0.0281573086977005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,089 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 072df2da98728631979ee233c5a94c13 2024-11-09T02:13:26,089 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 072df2da98728631979ee233c5a94c13: Running coprocessor pre-open hook at 1731118406058Writing region info on filesystem at 1731118406058Initializing all the Stores at 1731118406060 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406060Cleaning up temporary data from old regions at 1731118406078 (+18 ms)Running coprocessor post-open hooks at 1731118406089 (+11 ms)Region opened successfully at 1731118406089 2024-11-09T02:13:26,089 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:26,089 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:26,090 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:26,090 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1731118405247.072df2da98728631979ee233c5a94c13., pid=35, masterSystemTime=1731118405930 2024-11-09T02:13:26,092 INFO [StoreOpener-bb1296b406bf2a79b39a6de9a4389fcc-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:26,093 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=87e2ba48090bed7cebebc5b429271688, ASSIGN in 492 msec 2024-11-09T02:13:26,094 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:26,094 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/88ba7a90c8e825401b464926d88f3b36/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,094 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:26,094 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:26,094 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => ad814e81d5536ce802390dd5602e78a1, NAME => 'test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-09T02:13:26,095 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:26,095 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 88ba7a90c8e825401b464926d88f3b36; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73578370, jitterRate=0.09640315175056458}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,095 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,095 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:26,095 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:26,095 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:26,095 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 88ba7a90c8e825401b464926d88f3b36: Running coprocessor pre-open hook at 1731118406074Writing region info on filesystem at 1731118406074Initializing all the Stores at 1731118406076 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406076Cleaning up temporary data from old regions at 1731118406086 (+10 ms)Running coprocessor post-open hooks at 1731118406095 (+9 ms)Region opened successfully at 1731118406095 2024-11-09T02:13:26,095 INFO [StoreOpener-bb1296b406bf2a79b39a6de9a4389fcc-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bb1296b406bf2a79b39a6de9a4389fcc columnFamilyName family 2024-11-09T02:13:26,096 DEBUG [StoreOpener-bb1296b406bf2a79b39a6de9a4389fcc-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,096 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36., pid=56, masterSystemTime=1731118406032 2024-11-09T02:13:26,096 INFO [StoreOpener-bb1296b406bf2a79b39a6de9a4389fcc-1 {}] regionserver.HStore(327): Store=bb1296b406bf2a79b39a6de9a4389fcc/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,097 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=072df2da98728631979ee233c5a94c13, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:26,097 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:26,098 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:26,099 INFO [StoreOpener-ad814e81d5536ce802390dd5602e78a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:26,100 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:26,101 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:26,101 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure 072df2da98728631979ee233c5a94c13, server=53f286d3af3b,46883,1731118402441 because future has completed 2024-11-09T02:13:26,101 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:26,101 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:26,101 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:26,101 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:26,101 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => 85da2d651446630bfba59fcb89f923a1, NAME => 'test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-09T02:13:26,101 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=88ba7a90c8e825401b464926d88f3b36, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:26,102 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:26,102 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,102 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:26,102 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:26,104 INFO [StoreOpener-ad814e81d5536ce802390dd5602e78a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ad814e81d5536ce802390dd5602e78a1 columnFamilyName family 2024-11-09T02:13:26,105 DEBUG [StoreOpener-ad814e81d5536ce802390dd5602e78a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,33749,1731118402489 because future has completed 2024-11-09T02:13:26,105 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:26,105 INFO [StoreOpener-ad814e81d5536ce802390dd5602e78a1-1 {}] regionserver.HStore(327): Store=ad814e81d5536ce802390dd5602e78a1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,106 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:26,105 INFO [StoreOpener-85da2d651446630bfba59fcb89f923a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:26,107 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:26,108 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:26,109 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:26,109 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:26,109 INFO [StoreOpener-85da2d651446630bfba59fcb89f923a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85da2d651446630bfba59fcb89f923a1 columnFamilyName family 2024-11-09T02:13:26,109 DEBUG [StoreOpener-85da2d651446630bfba59fcb89f923a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,109 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/bb1296b406bf2a79b39a6de9a4389fcc/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,110 INFO [StoreOpener-85da2d651446630bfba59fcb89f923a1-1 {}] regionserver.HStore(327): Store=85da2d651446630bfba59fcb89f923a1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,110 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:26,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=11 2024-11-09T02:13:26,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure 072df2da98728631979ee233c5a94c13, server=53f286d3af3b,46883,1731118402441 in 303 msec 2024-11-09T02:13:26,111 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened bb1296b406bf2a79b39a6de9a4389fcc; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64620809, jitterRate=-0.03707490861415863}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,111 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:26,111 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for bb1296b406bf2a79b39a6de9a4389fcc: Running coprocessor pre-open hook at 1731118406090Writing region info on filesystem at 1731118406090Initializing all the Stores at 1731118406091 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406091Cleaning up temporary data from old regions at 1731118406101 (+10 ms)Running coprocessor post-open hooks at 1731118406111 (+10 ms)Region opened successfully at 1731118406111 2024-11-09T02:13:26,112 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:26,112 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:26,112 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:26,113 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc., pid=48, masterSystemTime=1731118405990 2024-11-09T02:13:26,113 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:26,113 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:26,115 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=072df2da98728631979ee233c5a94c13, ASSIGN in 514 msec 2024-11-09T02:13:26,116 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:26,116 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:26,117 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:26,117 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => f372b843955cc5a3ace82b165e5d33b0, NAME => 'test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-09T02:13:26,118 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:26,118 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:26,118 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:26,118 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,118 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:26,119 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:26,122 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bb1296b406bf2a79b39a6de9a4389fcc, server=53f286d3af3b,45941,1731118402318 because future has completed 2024-11-09T02:13:26,123 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/ad814e81d5536ce802390dd5602e78a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,124 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened ad814e81d5536ce802390dd5602e78a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63836637, jitterRate=-0.048759981989860535}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,124 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:26,125 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for ad814e81d5536ce802390dd5602e78a1: Running coprocessor pre-open hook at 1731118406095Writing region info on filesystem at 1731118406095Initializing all the Stores at 1731118406097 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406097Cleaning up temporary data from old regions at 1731118406109 (+12 ms)Running coprocessor post-open hooks at 1731118406124 (+15 ms)Region opened successfully at 1731118406125 (+1 ms) 2024-11-09T02:13:26,125 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/85da2d651446630bfba59fcb89f923a1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,126 INFO [StoreOpener-f372b843955cc5a3ace82b165e5d33b0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:26,126 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1., pid=36, masterSystemTime=1731118405930 2024-11-09T02:13:26,127 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened 85da2d651446630bfba59fcb89f923a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61752007, jitterRate=-0.07982338964939117}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,127 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:26,128 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for 85da2d651446630bfba59fcb89f923a1: Running coprocessor pre-open hook at 1731118406102Writing region info on filesystem at 1731118406102Initializing all the Stores at 1731118406104 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406104Cleaning up temporary data from old regions at 1731118406113 (+9 ms)Running coprocessor post-open hooks at 1731118406127 (+14 ms)Region opened successfully at 1731118406128 (+1 ms) 2024-11-09T02:13:26,128 INFO [StoreOpener-f372b843955cc5a3ace82b165e5d33b0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f372b843955cc5a3ace82b165e5d33b0 columnFamilyName family 2024-11-09T02:13:26,128 DEBUG [StoreOpener-f372b843955cc5a3ace82b165e5d33b0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,129 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1., pid=46, masterSystemTime=1731118406032 2024-11-09T02:13:26,129 INFO [StoreOpener-f372b843955cc5a3ace82b165e5d33b0-1 {}] regionserver.HStore(327): Store=f372b843955cc5a3ace82b165e5d33b0/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,130 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:26,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=5 2024-11-09T02:13:26,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bb1296b406bf2a79b39a6de9a4389fcc, server=53f286d3af3b,45941,1731118402318 in 234 msec 2024-11-09T02:13:26,131 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:26,131 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:26,131 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:26,131 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 555bb0a1c37bd4750a7abd799c897bc7, NAME => 'test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-09T02:13:26,131 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:26,132 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:26,132 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,132 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:26,132 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:26,132 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:26,133 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:26,133 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:26,133 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:26,133 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:26,133 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:26,133 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:26,135 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => bbfdf2901c997d044e2c20f81e3a2a90, NAME => 'test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-09T02:13:26,136 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:26,136 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure ad814e81d5536ce802390dd5602e78a1, server=53f286d3af3b,46883,1731118402441 because future has completed 2024-11-09T02:13:26,136 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:26,136 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:26,137 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=85da2d651446630bfba59fcb89f923a1, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:26,137 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:26,140 INFO [StoreOpener-555bb0a1c37bd4750a7abd799c897bc7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:26,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,33749,1731118402489 because future has completed 2024-11-09T02:13:26,142 INFO [StoreOpener-555bb0a1c37bd4750a7abd799c897bc7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 555bb0a1c37bd4750a7abd799c897bc7 columnFamilyName family 2024-11-09T02:13:26,142 DEBUG [StoreOpener-555bb0a1c37bd4750a7abd799c897bc7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,143 INFO [StoreOpener-555bb0a1c37bd4750a7abd799c897bc7-1 {}] regionserver.HStore(327): Store=555bb0a1c37bd4750a7abd799c897bc7/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,143 INFO [StoreOpener-bbfdf2901c997d044e2c20f81e3a2a90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:26,144 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:26,144 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/f372b843955cc5a3ace82b165e5d33b0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=16 2024-11-09T02:13:26,144 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure ad814e81d5536ce802390dd5602e78a1, server=53f286d3af3b,46883,1731118402441 in 331 msec 2024-11-09T02:13:26,145 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:26,145 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:26,145 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened f372b843955cc5a3ace82b165e5d33b0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68510250, jitterRate=0.02088227868080139}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,146 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:26,146 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for f372b843955cc5a3ace82b165e5d33b0: Running coprocessor pre-open hook at 1731118406119Writing region info on filesystem at 1731118406119Initializing all the Stores at 1731118406122 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406122Cleaning up temporary data from old regions at 1731118406133 (+11 ms)Running coprocessor post-open hooks at 1731118406146 (+13 ms)Region opened successfully at 1731118406146 2024-11-09T02:13:26,146 INFO [StoreOpener-bbfdf2901c997d044e2c20f81e3a2a90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbfdf2901c997d044e2c20f81e3a2a90 columnFamilyName family 2024-11-09T02:13:26,146 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:26,146 DEBUG [StoreOpener-bbfdf2901c997d044e2c20f81e3a2a90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,146 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:26,148 INFO [StoreOpener-bbfdf2901c997d044e2c20f81e3a2a90-1 {}] regionserver.HStore(327): Store=bbfdf2901c997d044e2c20f81e3a2a90/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,148 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0., pid=43, masterSystemTime=1731118405990 2024-11-09T02:13:26,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ad814e81d5536ce802390dd5602e78a1, ASSIGN in 548 msec 2024-11-09T02:13:26,150 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:26,152 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:26,152 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:26,153 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:26,153 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:26,153 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 923e0410f16fe611d9240651fc8bf39e, NAME => 'test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-09T02:13:26,153 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:26,153 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,154 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:26,154 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:26,154 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:26,155 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:26,155 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:26,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=29 2024-11-09T02:13:26,156 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:26,156 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,33749,1731118402489 in 272 msec 2024-11-09T02:13:26,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN in 559 msec 2024-11-09T02:13:26,158 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:26,160 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:26,160 INFO [StoreOpener-923e0410f16fe611d9240651fc8bf39e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:26,162 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/555bb0a1c37bd4750a7abd799c897bc7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,162 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/bbfdf2901c997d044e2c20f81e3a2a90/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,163 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 555bb0a1c37bd4750a7abd799c897bc7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64662980, jitterRate=-0.03644651174545288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,163 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:26,163 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 555bb0a1c37bd4750a7abd799c897bc7: Running coprocessor pre-open hook at 1731118406132Writing region info on filesystem at 1731118406132Initializing all the Stores at 1731118406134 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406134Cleaning up temporary data from old regions at 1731118406147 (+13 ms)Running coprocessor post-open hooks at 1731118406163 (+16 ms)Region opened successfully at 1731118406163 2024-11-09T02:13:26,163 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure f372b843955cc5a3ace82b165e5d33b0, server=53f286d3af3b,45941,1731118402318 because future has completed 2024-11-09T02:13:26,164 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened bbfdf2901c997d044e2c20f81e3a2a90; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63570558, jitterRate=-0.052724868059158325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,164 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:26,164 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for bbfdf2901c997d044e2c20f81e3a2a90: Running coprocessor pre-open hook at 1731118406136Writing region info on filesystem at 1731118406136Initializing all the Stores at 1731118406138 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406138Cleaning up temporary data from old regions at 1731118406156 (+18 ms)Running coprocessor post-open hooks at 1731118406164 (+8 ms)Region opened successfully at 1731118406164 2024-11-09T02:13:26,164 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7., pid=33, masterSystemTime=1731118405930 2024-11-09T02:13:26,164 INFO [StoreOpener-923e0410f16fe611d9240651fc8bf39e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 923e0410f16fe611d9240651fc8bf39e columnFamilyName family 2024-11-09T02:13:26,165 DEBUG [StoreOpener-923e0410f16fe611d9240651fc8bf39e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,165 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90., pid=47, masterSystemTime=1731118406032 2024-11-09T02:13:26,165 INFO [StoreOpener-923e0410f16fe611d9240651fc8bf39e-1 {}] regionserver.HStore(327): Store=923e0410f16fe611d9240651fc8bf39e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,166 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:26,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bb1296b406bf2a79b39a6de9a4389fcc, ASSIGN in 566 msec 2024-11-09T02:13:26,167 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:26,167 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:26,167 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:26,168 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => e33ccfa49d4634ee727e7cd03b929a72, NAME => 'test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-09T02:13:26,168 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:26,168 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:26,168 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,168 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:26,168 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:26,169 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:26,169 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:26,169 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:26,169 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:26,169 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:26,169 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:26,170 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:26,170 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => b3b3ba949727dd9291d8adc925fb09cb, NAME => 'test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-09T02:13:26,170 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:26,170 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,170 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:26,170 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:26,174 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure 555bb0a1c37bd4750a7abd799c897bc7, server=53f286d3af3b,46883,1731118402441 because future has completed 2024-11-09T02:13:26,174 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:26,175 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:26,178 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,33749,1731118402489 because future has completed 2024-11-09T02:13:26,180 INFO [StoreOpener-e33ccfa49d4634ee727e7cd03b929a72-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:26,181 INFO [StoreOpener-b3b3ba949727dd9291d8adc925fb09cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:26,181 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,45941,1731118402318, table=test, region=f372b843955cc5a3ace82b165e5d33b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,183 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/923e0410f16fe611d9240651fc8bf39e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,183 INFO [StoreOpener-b3b3ba949727dd9291d8adc925fb09cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3b3ba949727dd9291d8adc925fb09cb columnFamilyName family 2024-11-09T02:13:26,183 INFO [StoreOpener-e33ccfa49d4634ee727e7cd03b929a72-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e33ccfa49d4634ee727e7cd03b929a72 columnFamilyName family 2024-11-09T02:13:26,183 DEBUG [StoreOpener-b3b3ba949727dd9291d8adc925fb09cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,183 DEBUG [StoreOpener-e33ccfa49d4634ee727e7cd03b929a72-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,184 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 923e0410f16fe611d9240651fc8bf39e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58947168, jitterRate=-0.12161874771118164}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,184 INFO [StoreOpener-e33ccfa49d4634ee727e7cd03b929a72-1 {}] regionserver.HStore(327): Store=e33ccfa49d4634ee727e7cd03b929a72/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,184 INFO [StoreOpener-b3b3ba949727dd9291d8adc925fb09cb-1 {}] regionserver.HStore(327): Store=b3b3ba949727dd9291d8adc925fb09cb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,184 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:26,184 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:26,184 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 923e0410f16fe611d9240651fc8bf39e: Running coprocessor pre-open hook at 1731118406154Writing region info on filesystem at 1731118406154Initializing all the Stores at 1731118406156 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406156Cleaning up temporary data from old regions at 1731118406170 (+14 ms)Running coprocessor post-open hooks at 1731118406184 (+14 ms)Region opened successfully at 1731118406184 2024-11-09T02:13:26,184 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:26,186 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:26,186 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:26,187 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:26,187 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e., pid=40, masterSystemTime=1731118405990 2024-11-09T02:13:26,187 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:26,187 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:26,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=13 2024-11-09T02:13:26,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure 555bb0a1c37bd4750a7abd799c897bc7, server=53f286d3af3b,46883,1731118402441 in 389 msec 2024-11-09T02:13:26,193 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:26,196 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/b3b3ba949727dd9291d8adc925fb09cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,196 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:26,197 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:26,198 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:26,198 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,33749,1731118402489, table=test, region=8aa375f3cce51bfb890245ebbdc0e977. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,198 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:26,198 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,33749,1731118402489, table=test, region=88ba7a90c8e825401b464926d88f3b36. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,198 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:26,198 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,33749,1731118402489, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,198 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:26,198 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => 194b5628d516a374d3336a52e639da04, NAME => 'test,ttt,1731118405247.194b5628d516a374d3336a52e639da04.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-09T02:13:26,199 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 194b5628d516a374d3336a52e639da04 2024-11-09T02:13:26,199 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,ttt,1731118405247.194b5628d516a374d3336a52e639da04.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,199 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for 194b5628d516a374d3336a52e639da04 2024-11-09T02:13:26,199 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for 194b5628d516a374d3336a52e639da04 2024-11-09T02:13:26,199 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=555bb0a1c37bd4750a7abd799c897bc7, ASSIGN in 593 msec 2024-11-09T02:13:26,201 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:26,201 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:26,202 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened b3b3ba949727dd9291d8adc925fb09cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69374948, jitterRate=0.03376728296279907}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,202 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:26,202 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for b3b3ba949727dd9291d8adc925fb09cb: Running coprocessor pre-open hook at 1731118406170Writing region info on filesystem at 1731118406171 (+1 ms)Initializing all the Stores at 1731118406172 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406172Cleaning up temporary data from old regions at 1731118406187 (+15 ms)Running coprocessor post-open hooks at 1731118406202 (+15 ms)Region opened successfully at 1731118406202 2024-11-09T02:13:26,204 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb., pid=52, masterSystemTime=1731118406032 2024-11-09T02:13:26,205 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure 923e0410f16fe611d9240651fc8bf39e, server=53f286d3af3b,45941,1731118402318 because future has completed 2024-11-09T02:13:26,207 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:26,207 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:26,207 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:26,208 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a2a5d6e6a65b43660fcb665b4ef5c83, NAME => 'test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-09T02:13:26,208 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:26,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=26 2024-11-09T02:13:26,208 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,33749,1731118402489 in 321 msec 2024-11-09T02:13:26,209 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:26,209 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:26,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN in 612 msec 2024-11-09T02:13:26,213 INFO [StoreOpener-194b5628d516a374d3336a52e639da04-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 194b5628d516a374d3336a52e639da04 2024-11-09T02:13:26,214 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/e33ccfa49d4634ee727e7cd03b929a72/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,215 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened e33ccfa49d4634ee727e7cd03b929a72; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68779525, jitterRate=0.02489478886127472}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,215 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:26,215 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:26,215 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for e33ccfa49d4634ee727e7cd03b929a72: Running coprocessor pre-open hook at 1731118406169Writing region info on filesystem at 1731118406169Initializing all the Stores at 1731118406170 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406170Cleaning up temporary data from old regions at 1731118406198 (+28 ms)Running coprocessor post-open hooks at 1731118406215 (+17 ms)Region opened successfully at 1731118406215 2024-11-09T02:13:26,215 INFO [StoreOpener-194b5628d516a374d3336a52e639da04-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 194b5628d516a374d3336a52e639da04 columnFamilyName family 2024-11-09T02:13:26,215 DEBUG [StoreOpener-194b5628d516a374d3336a52e639da04-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,216 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72., pid=31, masterSystemTime=1731118405930 2024-11-09T02:13:26,216 INFO [StoreOpener-194b5628d516a374d3336a52e639da04-1 {}] regionserver.HStore(327): Store=194b5628d516a374d3336a52e639da04/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,216 INFO [StoreOpener-3a2a5d6e6a65b43660fcb665b4ef5c83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:26,217 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for 194b5628d516a374d3336a52e639da04 2024-11-09T02:13:26,218 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/194b5628d516a374d3336a52e639da04 2024-11-09T02:13:26,219 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/194b5628d516a374d3336a52e639da04 2024-11-09T02:13:26,219 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,33749,1731118402489 because future has completed 2024-11-09T02:13:26,220 INFO [StoreOpener-3a2a5d6e6a65b43660fcb665b4ef5c83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a2a5d6e6a65b43660fcb665b4ef5c83 columnFamilyName family 2024-11-09T02:13:26,220 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for 194b5628d516a374d3336a52e639da04 2024-11-09T02:13:26,220 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for 194b5628d516a374d3336a52e639da04 2024-11-09T02:13:26,220 DEBUG [StoreOpener-3a2a5d6e6a65b43660fcb665b4ef5c83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,221 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:26,221 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:26,221 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:26,221 INFO [StoreOpener-3a2a5d6e6a65b43660fcb665b4ef5c83-1 {}] regionserver.HStore(327): Store=3a2a5d6e6a65b43660fcb665b4ef5c83/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,222 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:26,222 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f3f9d6837573933dea12bd60465775e, NAME => 'test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-09T02:13:26,222 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:26,222 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,222 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:26,222 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:26,223 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:26,223 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for 194b5628d516a374d3336a52e639da04 2024-11-09T02:13:26,223 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:26,224 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:26,224 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:26,225 INFO [StoreOpener-5f3f9d6837573933dea12bd60465775e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:26,226 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=17 2024-11-09T02:13:26,227 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,33749,1731118402489 in 310 msec 2024-11-09T02:13:26,228 INFO [StoreOpener-5f3f9d6837573933dea12bd60465775e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f3f9d6837573933dea12bd60465775e columnFamilyName family 2024-11-09T02:13:26,228 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/194b5628d516a374d3336a52e639da04/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,228 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:26,228 DEBUG [StoreOpener-5f3f9d6837573933dea12bd60465775e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,228 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened 194b5628d516a374d3336a52e639da04; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66761876, jitterRate=-0.0051705241203308105}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,229 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 194b5628d516a374d3336a52e639da04 2024-11-09T02:13:26,229 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for 194b5628d516a374d3336a52e639da04: Running coprocessor pre-open hook at 1731118406199Writing region info on filesystem at 1731118406199Initializing all the Stores at 1731118406200 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406200Cleaning up temporary data from old regions at 1731118406220 (+20 ms)Running coprocessor post-open hooks at 1731118406229 (+9 ms)Region opened successfully at 1731118406229 2024-11-09T02:13:26,229 INFO [StoreOpener-5f3f9d6837573933dea12bd60465775e-1 {}] regionserver.HStore(327): Store=5f3f9d6837573933dea12bd60465775e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,230 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:26,231 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:26,231 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN in 630 msec 2024-11-09T02:13:26,231 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1731118405247.194b5628d516a374d3336a52e639da04., pid=42, masterSystemTime=1731118405990 2024-11-09T02:13:26,232 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:26,233 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:26,233 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:26,234 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:26,234 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:26,234 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:26,234 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:26,235 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => 5c24783f69139da02a463f7c484b941b, NAME => 'test,eee,1731118405247.5c24783f69139da02a463f7c484b941b.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-09T02:13:26,235 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:26,235 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,eee,1731118405247.5c24783f69139da02a463f7c484b941b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,235 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for 5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:26,235 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for 5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:26,237 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:26,238 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure e33ccfa49d4634ee727e7cd03b929a72, server=53f286d3af3b,46883,1731118402441 because future has completed 2024-11-09T02:13:26,238 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/3a2a5d6e6a65b43660fcb665b4ef5c83/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,238 INFO [StoreOpener-5c24783f69139da02a463f7c484b941b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:26,239 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 3a2a5d6e6a65b43660fcb665b4ef5c83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61760104, jitterRate=-0.07970273494720459}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,239 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:26,239 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 3a2a5d6e6a65b43660fcb665b4ef5c83: Running coprocessor pre-open hook at 1731118406209Writing region info on filesystem at 1731118406209Initializing all the Stores at 1731118406211 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406211Cleaning up temporary data from old regions at 1731118406224 (+13 ms)Running coprocessor post-open hooks at 1731118406239 (+15 ms)Region opened successfully at 1731118406239 2024-11-09T02:13:26,240 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83., pid=51, masterSystemTime=1731118406032 2024-11-09T02:13:26,240 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=194b5628d516a374d3336a52e639da04, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:26,241 INFO [StoreOpener-5c24783f69139da02a463f7c484b941b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5c24783f69139da02a463f7c484b941b columnFamilyName family 2024-11-09T02:13:26,241 DEBUG [StoreOpener-5c24783f69139da02a463f7c484b941b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,243 INFO [StoreOpener-5c24783f69139da02a463f7c484b941b-1 {}] regionserver.HStore(327): Store=5c24783f69139da02a463f7c484b941b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,243 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:26,243 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:26,243 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 194b5628d516a374d3336a52e639da04, server=53f286d3af3b,45941,1731118402318 because future has completed 2024-11-09T02:13:26,243 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:26,243 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for 5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:26,243 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 8fe76b2cbbde488b7620457c1f89dcc9, NAME => 'test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-09T02:13:26,244 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:26,244 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,244 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:26,244 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:26,245 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:26,245 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/5f3f9d6837573933dea12bd60465775e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,246 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:26,246 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:26,246 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 5f3f9d6837573933dea12bd60465775e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75419198, jitterRate=0.12383362650871277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,246 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:26,246 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 5f3f9d6837573933dea12bd60465775e: Running coprocessor pre-open hook at 1731118406223Writing region info on filesystem at 1731118406223Initializing all the Stores at 1731118406224 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406224Cleaning up temporary data from old regions at 1731118406233 (+9 ms)Running coprocessor post-open hooks at 1731118406246 (+13 ms)Region opened successfully at 1731118406246 2024-11-09T02:13:26,247 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for 5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:26,247 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for 5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:26,249 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e., pid=34, masterSystemTime=1731118405930 2024-11-09T02:13:26,249 INFO [StoreOpener-8fe76b2cbbde488b7620457c1f89dcc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:26,250 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for 5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:26,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a2a5d6e6a65b43660fcb665b4ef5c83, server=53f286d3af3b,33749,1731118402489 because future has completed 2024-11-09T02:13:26,252 INFO [StoreOpener-8fe76b2cbbde488b7620457c1f89dcc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fe76b2cbbde488b7620457c1f89dcc9 columnFamilyName family 2024-11-09T02:13:26,252 DEBUG [StoreOpener-8fe76b2cbbde488b7620457c1f89dcc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,253 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:26,253 INFO [StoreOpener-8fe76b2cbbde488b7620457c1f89dcc9-1 {}] regionserver.HStore(327): Store=8fe76b2cbbde488b7620457c1f89dcc9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,253 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:26,253 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:26,255 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:26,255 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:26,256 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:26,256 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:26,257 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/5c24783f69139da02a463f7c484b941b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=24 2024-11-09T02:13:26,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 194b5628d516a374d3336a52e639da04, server=53f286d3af3b,45941,1731118402318 in 405 msec 2024-11-09T02:13:26,258 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened 5c24783f69139da02a463f7c484b941b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74595096, jitterRate=0.11155354976654053}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,258 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:26,258 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for 5c24783f69139da02a463f7c484b941b: Running coprocessor pre-open hook at 1731118406235Writing region info on filesystem at 1731118406235Initializing all the Stores at 1731118406237 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406237Cleaning up temporary data from old regions at 1731118406247 (+10 ms)Running coprocessor post-open hooks at 1731118406258 (+11 ms)Region opened successfully at 1731118406258 2024-11-09T02:13:26,259 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1731118405247.5c24783f69139da02a463f7c484b941b., pid=49, masterSystemTime=1731118405990 2024-11-09T02:13:26,260 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:26,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=194b5628d516a374d3336a52e639da04, ASSIGN in 661 msec 2024-11-09T02:13:26,262 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:26,262 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:26,263 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:26,263 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 6e77447186c114b3577a6f77b1ffe798, NAME => 'test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-09T02:13:26,263 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:26,263 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,263 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:26,264 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:26,264 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=5c24783f69139da02a463f7c484b941b, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:26,266 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8fe76b2cbbde488b7620457c1f89dcc9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,266 INFO [StoreOpener-6e77447186c114b3577a6f77b1ffe798-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:26,267 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 8fe76b2cbbde488b7620457c1f89dcc9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74667536, jitterRate=0.11263298988342285}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,267 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:26,267 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 8fe76b2cbbde488b7620457c1f89dcc9: Running coprocessor pre-open hook at 1731118406244Writing region info on filesystem at 1731118406244Initializing all the Stores at 1731118406246 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406246Cleaning up temporary data from old regions at 1731118406256 (+10 ms)Running coprocessor post-open hooks at 1731118406267 (+11 ms)Region opened successfully at 1731118406267 2024-11-09T02:13:26,267 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5c24783f69139da02a463f7c484b941b, server=53f286d3af3b,45941,1731118402318 because future has completed 2024-11-09T02:13:26,268 INFO [StoreOpener-6e77447186c114b3577a6f77b1ffe798-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6e77447186c114b3577a6f77b1ffe798 columnFamilyName family 2024-11-09T02:13:26,268 DEBUG [StoreOpener-6e77447186c114b3577a6f77b1ffe798-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,268 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9., pid=55, masterSystemTime=1731118406032 2024-11-09T02:13:26,269 INFO [StoreOpener-6e77447186c114b3577a6f77b1ffe798-1 {}] regionserver.HStore(327): Store=6e77447186c114b3577a6f77b1ffe798/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,269 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:26,270 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:26,271 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:26,271 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:26,271 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:26,271 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:26,271 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 195f23de0822a5bc7d5a1a2b4b16afdb, NAME => 'test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-09T02:13:26,272 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:26,272 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:26,272 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:26,272 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:26,272 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:26,272 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:26,274 INFO [StoreOpener-195f23de0822a5bc7d5a1a2b4b16afdb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:26,275 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:26,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=9 2024-11-09T02:13:26,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 5c24783f69139da02a463f7c484b941b, server=53f286d3af3b,45941,1731118402318 in 376 msec 2024-11-09T02:13:26,276 INFO [StoreOpener-195f23de0822a5bc7d5a1a2b4b16afdb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 195f23de0822a5bc7d5a1a2b4b16afdb columnFamilyName family 2024-11-09T02:13:26,276 DEBUG [StoreOpener-195f23de0822a5bc7d5a1a2b4b16afdb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:26,277 INFO [StoreOpener-195f23de0822a5bc7d5a1a2b4b16afdb-1 {}] regionserver.HStore(327): Store=195f23de0822a5bc7d5a1a2b4b16afdb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:26,277 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:26,279 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:26,279 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/6e77447186c114b3577a6f77b1ffe798/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,279 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5c24783f69139da02a463f7c484b941b, ASSIGN in 678 msec 2024-11-09T02:13:26,280 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:26,280 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:26,280 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:26,280 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 6e77447186c114b3577a6f77b1ffe798; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72901248, jitterRate=0.08631324768066406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,281 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:26,281 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 6e77447186c114b3577a6f77b1ffe798: Running coprocessor pre-open hook at 1731118406264Writing region info on filesystem at 1731118406264Initializing all the Stores at 1731118406265 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406265Cleaning up temporary data from old regions at 1731118406272 (+7 ms)Running coprocessor post-open hooks at 1731118406281 (+9 ms)Region opened successfully at 1731118406281 2024-11-09T02:13:26,282 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:26,283 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798., pid=44, masterSystemTime=1731118405990 2024-11-09T02:13:26,284 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:26,286 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,45941,1731118402318, table=test, region=923e0410f16fe611d9240651fc8bf39e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,286 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,33749,1731118402489 because future has completed 2024-11-09T02:13:26,286 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,45941,1731118402318, table=test, region=f372b843955cc5a3ace82b165e5d33b0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,287 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:26,288 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:26,288 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:26,290 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,46883,1731118402441, table=test, region=5f3f9d6837573933dea12bd60465775e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,290 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,46883,1731118402441, table=test, region=e33ccfa49d4634ee727e7cd03b929a72. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,291 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f3f9d6837573933dea12bd60465775e, server=53f286d3af3b,46883,1731118402441 because future has completed 2024-11-09T02:13:26,293 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/195f23de0822a5bc7d5a1a2b4b16afdb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-09T02:13:26,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:26,294 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 195f23de0822a5bc7d5a1a2b4b16afdb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61719022, jitterRate=-0.08031490445137024}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:26,294 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:26,294 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 195f23de0822a5bc7d5a1a2b4b16afdb: Running coprocessor pre-open hook at 1731118406273Writing region info on filesystem at 1731118406273Initializing all the Stores at 1731118406274 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118406274Cleaning up temporary data from old regions at 1731118406280 (+6 ms)Running coprocessor post-open hooks at 1731118406294 (+14 ms)Region opened successfully at 1731118406294 2024-11-09T02:13:26,295 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb., pid=53, masterSystemTime=1731118406032 2024-11-09T02:13:26,296 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6e77447186c114b3577a6f77b1ffe798, server=53f286d3af3b,45941,1731118402318 because future has completed 2024-11-09T02:13:26,298 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:26,298 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:26,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=8 2024-11-09T02:13:26,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f3f9d6837573933dea12bd60465775e, server=53f286d3af3b,46883,1731118402441 in 501 msec 2024-11-09T02:13:26,302 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,33749,1731118402489, table=test, region=8aa375f3cce51bfb890245ebbdc0e977. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,302 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,33749,1731118402489, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,302 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,33749,1731118402489, table=test, region=88ba7a90c8e825401b464926d88f3b36. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,302 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,33749,1731118402489, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:26,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5f3f9d6837573933dea12bd60465775e, ASSIGN in 704 msec 2024-11-09T02:13:26,305 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,33749,1731118402489 2024-11-09T02:13:26,308 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,33749,1731118402489 because future has completed 2024-11-09T02:13:26,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=30 2024-11-09T02:13:26,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 6e77447186c114b3577a6f77b1ffe798, server=53f286d3af3b,45941,1731118402318 in 450 msec 2024-11-09T02:13:26,313 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6e77447186c114b3577a6f77b1ffe798, ASSIGN in 714 msec 2024-11-09T02:13:26,316 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=6 2024-11-09T02:13:26,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,33749,1731118402489 in 397 msec 2024-11-09T02:13:26,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN in 720 msec 2024-11-09T02:13:26,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=14 2024-11-09T02:13:26,323 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,33749,1731118402489 in 391 msec 2024-11-09T02:13:26,326 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN in 726 msec 2024-11-09T02:13:26,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=22 2024-11-09T02:13:26,330 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 3a2a5d6e6a65b43660fcb665b4ef5c83, server=53f286d3af3b,33749,1731118402489 in 419 msec 2024-11-09T02:13:26,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN in 733 msec 2024-11-09T02:13:26,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=28 2024-11-09T02:13:26,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure e33ccfa49d4634ee727e7cd03b929a72, server=53f286d3af3b,46883,1731118402441 in 558 msec 2024-11-09T02:13:26,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=e33ccfa49d4634ee727e7cd03b929a72, ASSIGN in 740 msec 2024-11-09T02:13:26,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=21 2024-11-09T02:13:26,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure 923e0410f16fe611d9240651fc8bf39e, server=53f286d3af3b,45941,1731118402318 in 504 msec 2024-11-09T02:13:26,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=923e0410f16fe611d9240651fc8bf39e, ASSIGN in 747 msec 2024-11-09T02:13:26,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=27 2024-11-09T02:13:26,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure f372b843955cc5a3ace82b165e5d33b0, server=53f286d3af3b,45941,1731118402318 in 495 msec 2024-11-09T02:13:26,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f372b843955cc5a3ace82b165e5d33b0, ASSIGN in 754 msec 2024-11-09T02:13:26,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=19 2024-11-09T02:13:26,357 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,33749,1731118402489 in 418 msec 2024-11-09T02:13:26,359 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN in 760 msec 2024-11-09T02:13:26,361 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=244d470b9561d8921c93364fa1596bac, regionState=OPEN, openSeqNum=2, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:26,363 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 244d470b9561d8921c93364fa1596bac, server=53f286d3af3b,45941,1731118402318 because future has completed 2024-11-09T02:13:26,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=10 2024-11-09T02:13:26,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,33749,1731118402489 in 442 msec 2024-11-09T02:13:26,367 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN in 768 msec 2024-11-09T02:13:26,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=18 2024-11-09T02:13:26,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 244d470b9561d8921c93364fa1596bac, server=53f286d3af3b,45941,1731118402318 in 526 msec 2024-11-09T02:13:26,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=4 2024-11-09T02:13:26,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=244d470b9561d8921c93364fa1596bac, ASSIGN in 773 msec 2024-11-09T02:13:26,375 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-09T02:13:26,375 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731118406375"}]},"ts":"1731118406375"} 2024-11-09T02:13:26,377 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-09T02:13:26,379 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-09T02:13:26,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.1200 sec 2024-11-09T02:13:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-09T02:13:26,414 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-09T02:13:26,414 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-09T02:13:26,416 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:13:26,437 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-09T02:13:26,438 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:13:26,439 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-09T02:13:26,439 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:13:26,445 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-09T02:13:26,472 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-09T02:13:26,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:13:26,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41818, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:13:26,497 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:13:26,499 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45836, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:13:26,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 195f23de0822a5bc7d5a1a2b4b16afdb changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8aa375f3cce51bfb890245ebbdc0e977 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8fe76b2cbbde488b7620457c1f89dcc9 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region b3b3ba949727dd9291d8adc925fb09cb changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 88ba7a90c8e825401b464926d88f3b36 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 3a2a5d6e6a65b43660fcb665b4ef5c83 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bbfdf2901c997d044e2c20f81e3a2a90 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 85da2d651446630bfba59fcb89f923a1 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region bb1296b406bf2a79b39a6de9a4389fcc changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5c24783f69139da02a463f7c484b941b changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6dd1285d59409a10928f32885afd0b51 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region a5a881b27904120149acda46e3d36994 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 244d470b9561d8921c93364fa1596bac changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 923e0410f16fe611d9240651fc8bf39e changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 194b5628d516a374d3336a52e639da04 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f372b843955cc5a3ace82b165e5d33b0 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6e77447186c114b3577a6f77b1ffe798 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f28f1a8d6b5be48d6e6dca7a81da9c57 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5f3f9d6837573933dea12bd60465775e changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 072df2da98728631979ee233c5a94c13 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 555bb0a1c37bd4750a7abd799c897bc7 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ad814e81d5536ce802390dd5602e78a1 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 7cb62548224e02c76c614c52af2aadca changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c5c32774defefb99b01b165b020a41fa changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 87e2ba48090bed7cebebc5b429271688 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region e33ccfa49d4634ee727e7cd03b929a72 changed from -1.0 to 0.0, refreshing cache 2024-11-09T02:13:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-09T02:13:26,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-09T02:13:26,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-09T02:13:26,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BalancerClusterState(204): Hosts are {53f286d3af3b=0} racks are {/default-rack=0} 2024-11-09T02:13:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BalancerClusterState(303): server 0 has 9 regions 2024-11-09T02:13:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BalancerClusterState(303): server 1 has 9 regions 2024-11-09T02:13:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-09T02:13:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T02:13:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T02:13:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-09T02:13:26,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T02:13:26,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T02:13:26,562 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-09T02:13:26,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-09T02:13:26,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] balancer.StochasticLoadBalancer(395): Cluster wide - skipping load balancing because weighted average imbalance=0.016474464579901153 <= threshold(0.025). If you want more aggressive balancing, either lower hbase.master.balancer.stochastic.minCostNeedBalance from 0.025 or increase the relative multiplier(s) of the specific cost function(s). functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-09T02:13:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-09T02:13:26,569 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:13:26,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-11-09T02:13:26,589 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=374, OpenFileDescriptor=619, MaxFileDescriptor=1048576, SystemLoadAverage=319, ProcessCount=11, AvailableMemoryMB=2899 2024-11-09T02:13:26,592 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-09T02:13:26,593 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-09T02:13:26,593 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 53f286d3af3b,33749,1731118402489 2024-11-09T02:13:26,594 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 53f286d3af3b,33749,1731118402489: Simulated kill ***** 2024-11-09T02:13:26,594 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-09T02:13:26,597 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-09T02:13:26,600 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-09T02:13:26,600 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-09T02:13:26,600 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-09T02:13:26,601 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 288359040 }, "NonHeapMemoryUsage": { "committed": 150667264, "init": 7667712, "max": -1, "used": 148426640 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "53f286d3af3b" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "53f286d3af3b" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "53f286d3af3b" } ] } 2024-11-09T02:13:26,604 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36011 {}] master.MasterRpcServices(700): 53f286d3af3b,33749,1731118402489 reported a fatal error: ***** ABORTING region server 53f286d3af3b,33749,1731118402489: Simulated kill ***** 2024-11-09T02:13:26,606 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '53f286d3af3b,33749,1731118402489' ***** 2024-11-09T02:13:26,606 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-09T02:13:26,606 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T02:13:26,607 INFO [RS:2;53f286d3af3b:33749 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-09T02:13:26,607 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T02:13:26,607 INFO [RS:2;53f286d3af3b:33749 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-09T02:13:26,607 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:13:26,607 INFO [RS:2;53f286d3af3b:33749 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T02:13:26,607 INFO [RS:2;53f286d3af3b:33749 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;53f286d3af3b:33749. 2024-11-09T02:13:26,607 DEBUG [RS:2;53f286d3af3b:33749 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:13:26,611 DEBUG [RS:2;53f286d3af3b:33749 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:13:26,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741836_1012 (size=32114) 2024-11-09T02:13:26,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741836_1012 (size=32114) 2024-11-09T02:13:26,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741835_1011 (size=2237) 2024-11-09T02:13:26,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741835_1011 (size=2237) 2024-11-09T02:13:26,625 DEBUG [RS:2;53f286d3af3b:33749 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:13:26,625 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T02:13:26,625 INFO [RS:2;53f286d3af3b:33749 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33749 2024-11-09T02:13:26,643 INFO [regionserver/53f286d3af3b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T02:13:26,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/53f286d3af3b,33749,1731118402489 2024-11-09T02:13:26,692 INFO [RS:2;53f286d3af3b:33749 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T02:13:26,692 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T02:13:26,703 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [53f286d3af3b,33749,1731118402489] 2024-11-09T02:13:26,713 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/53f286d3af3b,33749,1731118402489 already deleted, retry=false 2024-11-09T02:13:26,713 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 53f286d3af3b,33749,1731118402489 on 53f286d3af3b,36011,1731118401306 2024-11-09T02:13:26,718 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:26,722 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=57 for 53f286d3af3b,33749,1731118402489 (carryingMeta=true) 53f286d3af3b,33749,1731118402489/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@62ee72d7[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-09T02:13:26,723 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=57, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:26,724 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true, isMeta: true 2024-11-09T02:13:26,726 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting 2024-11-09T02:13:26,728 INFO [PEWorker-1 {}] master.SplitWALManager(105): 53f286d3af3b,33749,1731118402489 WAL count=1, meta=true 2024-11-09T02:13:26,730 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta}] 2024-11-09T02:13:26,737 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:26,740 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:26,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:13:26,803 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33749-0x1011d5e7e540003, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:13:26,803 INFO [RS:2;53f286d3af3b:33749 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T02:13:26,803 INFO [RS:2;53f286d3af3b:33749 {}] regionserver.HRegionServer(1031): Exiting; stopping=53f286d3af3b,33749,1731118402489; zookeeper connection closed. 2024-11-09T02:13:26,804 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@65f9dfe9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@65f9dfe9 2024-11-09T02:13:26,808 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-09T02:13:26,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45941 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=59 2024-11-09T02:13:26,929 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(299): Splitting hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, size=31.4 K (32114bytes) 2024-11-09T02:13:26,930 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-09T02:13:26,930 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-09T02:13:26,930 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta 2024-11-09T02:13:26,934 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta after 3ms 2024-11-09T02:13:26,942 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(310): Open hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta took 14ms 2024-11-09T02:13:26,953 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-09T02:13:26,957 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta.temp 2024-11-09T02:13:26,960 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta.temp 2024-11-09T02:13:26,963 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta so closing down 2024-11-09T02:13:26,964 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-09T02:13:26,964 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.OutputSink(145): 3 split writer threads finished 2024-11-09T02:13:26,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741864_1040 (size=31873) 2024-11-09T02:13:26,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741864_1040 (size=31873) 2024-11-09T02:13:26,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741864_1040 (size=31873) 2024-11-09T02:13:26,978 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) 2024-11-09T02:13:26,980 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta.temp to hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-09T02:13:26,984 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(425): Processed 58 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false 2024-11-09T02:13:26,984 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] wal.WALSplitter(428): Completed split of hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, journal: Splitting hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, size=31.4 K (32114bytes) at 1731118406929Creating recovered edits writer path=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta.temp at 1731118406960 (+31 ms)Finishing writing output for hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta so closing down at 1731118406964 (+4 ms)3 split writer threads finished at 1731118406964Closed recovered edits writer path=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta.temp (wrote 57 edits, skipped 0 edits in 2 ms) at 1731118406978 (+14 ms)Rename recovered edits hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta.temp to hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 at 1731118406981 (+3 ms)Processed 58 edits across 1 Regions in 37 ms; skipped=1; WAL=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, size=31.4 K, length=32114, corrupted=false, cancelled=false at 1731118406984 (+3 ms) 2024-11-09T02:13:26,984 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta 2024-11-09T02:13:26,986 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-09T02:13:26,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-09T02:13:26,996 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta to hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/oldWALs 2024-11-09T02:13:27,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-09T02:13:27,000 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 in 256 msec 2024-11-09T02:13:27,002 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:27,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-09T02:13:27,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 in 273 msec 2024-11-09T02:13:27,009 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting dir is empty, no logs to split. 2024-11-09T02:13:27,009 INFO [PEWorker-1 {}] master.SplitWALManager(105): 53f286d3af3b,33749,1731118402489 WAL count=0, meta=true 2024-11-09T02:13:27,009 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 53f286d3af3b,33749,1731118402489 WAL splitting is done? wals=0, meta=true 2024-11-09T02:13:27,012 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-09T02:13:27,013 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:27,015 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T02:13:27,133 INFO [regionserver/53f286d3af3b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-09T02:13:27,134 INFO [regionserver/53f286d3af3b:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-09T02:13:27,165 DEBUG [53f286d3af3b:36011 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-09T02:13:27,165 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(204): Hosts are {53f286d3af3b=0} racks are {/default-rack=0} 2024-11-09T02:13:27,166 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T02:13:27,166 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T02:13:27,166 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T02:13:27,166 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T02:13:27,166 INFO [53f286d3af3b:36011 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T02:13:27,166 INFO [53f286d3af3b:36011 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T02:13:27,166 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T02:13:27,166 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:27,168 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPENING 2024-11-09T02:13:27,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:27,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:27,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:27,219 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=60, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:27,219 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:27,219 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:27,219 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:27,219 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:27,381 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-09T02:13:27,381 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-09T02:13:27,381 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-09T02:13:27,385 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=53f286d3af3b%2C45941%2C1731118402318.meta, suffix=.meta, logDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,45941,1731118402318, archiveDir=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/oldWALs, maxLogs=32 2024-11-09T02:13:27,400 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,45941,1731118402318/53f286d3af3b%2C45941%2C1731118402318.meta.1731118407387.meta, exclude list is [], retry=0 2024-11-09T02:13:27,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39093,DS-5399098e-be1d-4c86-8665-bb7420bd7139,DISK] 2024-11-09T02:13:27,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43193,DS-1e0d34a9-ddd0-4293-9074-b2273601121a,DISK] 2024-11-09T02:13:27,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45399,DS-04930e57-9ab7-4098-82a3-72b98471424f,DISK] 2024-11-09T02:13:27,406 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,45941,1731118402318/53f286d3af3b%2C45941%2C1731118402318.meta.1731118407387.meta 2024-11-09T02:13:27,406 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41983:41983),(127.0.0.1/127.0.0.1:35315:35315),(127.0.0.1/127.0.0.1:42707:42707)] 2024-11-09T02:13:27,406 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-09T02:13:27,406 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-09T02:13:27,406 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-09T02:13:27,407 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-09T02:13:27,407 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-09T02:13:27,407 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:27,407 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-09T02:13:27,407 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-09T02:13:27,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-09T02:13:27,410 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-09T02:13:27,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:27,410 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:27,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-09T02:13:27,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-09T02:13:27,411 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:27,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:27,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-09T02:13:27,413 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-09T02:13:27,413 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:27,413 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:27,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-09T02:13:27,415 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-09T02:13:27,415 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:27,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-09T02:13:27,416 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-09T02:13:27,416 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740 2024-11-09T02:13:27,418 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740 2024-11-09T02:13:27,419 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-09T02:13:27,434 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5793): Applied 264, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=60, path=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-09T02:13:27,437 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=28 KB heapSize=48.31 KB 2024-11-09T02:13:27,512 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/info/725db6b0326744c2ba14053c5ed5b967 is 138, key is test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb./info:regioninfo/1731118406305/Put/seqid=0 2024-11-09T02:13:27,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741866_1042 (size=34924) 2024-11-09T02:13:27,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741866_1042 (size=34924) 2024-11-09T02:13:27,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741866_1042 (size=34924) 2024-11-09T02:13:27,524 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.86 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/info/725db6b0326744c2ba14053c5ed5b967 2024-11-09T02:13:27,583 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/ns/8259903d34b04eb382390e3d44b8475c is 43, key is default/ns:d/1731118404965/Put/seqid=0 2024-11-09T02:13:27,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741867_1043 (size=5153) 2024-11-09T02:13:27,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741867_1043 (size=5153) 2024-11-09T02:13:27,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741867_1043 (size=5153) 2024-11-09T02:13:27,595 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/ns/8259903d34b04eb382390e3d44b8475c 2024-11-09T02:13:27,620 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/table/c1302a45a9534181be6704793ae2f5b3 is 40, key is test/table:state/1731118406375/Put/seqid=0 2024-11-09T02:13:27,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741868_1044 (size=5165) 2024-11-09T02:13:27,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741868_1044 (size=5165) 2024-11-09T02:13:27,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741868_1044 (size=5165) 2024-11-09T02:13:27,630 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/table/c1302a45a9534181be6704793ae2f5b3 2024-11-09T02:13:27,640 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/info/725db6b0326744c2ba14053c5ed5b967 as hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/info/725db6b0326744c2ba14053c5ed5b967 2024-11-09T02:13:27,650 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/info/725db6b0326744c2ba14053c5ed5b967, entries=260, sequenceid=60, filesize=34.1 K 2024-11-09T02:13:27,654 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/ns/8259903d34b04eb382390e3d44b8475c as hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/ns/8259903d34b04eb382390e3d44b8475c 2024-11-09T02:13:27,663 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/ns/8259903d34b04eb382390e3d44b8475c, entries=2, sequenceid=60, filesize=5.0 K 2024-11-09T02:13:27,665 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/table/c1302a45a9534181be6704793ae2f5b3 as hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/table/c1302a45a9534181be6704793ae2f5b3 2024-11-09T02:13:27,676 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/table/c1302a45a9534181be6704793ae2f5b3, entries=2, sequenceid=60, filesize=5.0 K 2024-11-09T02:13:27,680 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 242ms, sequenceid=60, compaction requested=false; wal=null 2024-11-09T02:13:27,681 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-09T02:13:27,684 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/0000000000000000060 2024-11-09T02:13:27,686 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-09T02:13:27,686 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-09T02:13:27,687 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-09T02:13:27,689 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-09T02:13:27,693 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/60.seqid, newMaxSeqId=60, maxSeqId=1 2024-11-09T02:13:27,694 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=61; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63424954, jitterRate=-0.05489453673362732}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-09T02:13:27,694 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-09T02:13:27,695 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731118407407Writing region info on filesystem at 1731118407407Initializing all the Stores at 1731118407408 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118407409 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118407409Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118407409Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731118407409Obtaining lock to block concurrent updates at 1731118407437 (+28 ms)Preparing flush snapshotting stores in 1588230740 at 1731118407437Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=28672, getHeapSize=49408, getOffHeapSize=0, getCellsCount=264 at 1731118407441 (+4 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731118407441Flushing 1588230740/info: creating writer at 1731118407443 (+2 ms)Flushing 1588230740/info: appending metadata at 1731118407503 (+60 ms)Flushing 1588230740/info: closing flushed file at 1731118407505 (+2 ms)Flushing 1588230740/ns: creating writer at 1731118407567 (+62 ms)Flushing 1588230740/ns: appending metadata at 1731118407582 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731118407582Flushing 1588230740/table: creating writer at 1731118407604 (+22 ms)Flushing 1588230740/table: appending metadata at 1731118407619 (+15 ms)Flushing 1588230740/table: closing flushed file at 1731118407619Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1af6aaa4: reopening flushed file at 1731118407638 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3937aabf: reopening flushed file at 1731118407653 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2901c704: reopening flushed file at 1731118407664 (+11 ms)Finished flush of dataSize ~28 KB/28672, heapSize ~48.02 KB/49168, currentSize=0 B/0 for 1588230740 in 242ms, sequenceid=60, compaction requested=false; wal=null at 1731118407680 (+16 ms)Cleaning up temporary data from old regions at 1731118407686 (+6 ms)Running coprocessor post-open hooks at 1731118407694 (+8 ms)Region opened successfully at 1731118407694 2024-11-09T02:13:27,696 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=61, masterSystemTime=1731118407372 2024-11-09T02:13:27,699 DEBUG [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-09T02:13:27,699 INFO [RS_OPEN_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_META, pid=61}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-09T02:13:27,700 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=61, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:27,701 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:27,754 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-09T02:13:27,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:27,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:27,755 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:27,755 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=61, ppid=60, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:27,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:27,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:27,756 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:27,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-09T02:13:27,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 in 537 msec 2024-11-09T02:13:27,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=57 2024-11-09T02:13:27,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 748 msec 2024-11-09T02:13:27,762 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 53f286d3af3b,33749,1731118402489 had 9 regions 2024-11-09T02:13:27,764 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=57, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true, isMeta: false 2024-11-09T02:13:27,766 INFO [PEWorker-1 {}] master.SplitWALManager(105): 53f286d3af3b,33749,1731118402489 WAL count=1, meta=false 2024-11-09T02:13:27,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=57, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218}] 2024-11-09T02:13:27,769 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:27,770 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:27,923 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46883 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=63 2024-11-09T02:13:27,938 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(299): Splitting hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218, size=2.2 K (2237bytes) 2024-11-09T02:13:27,938 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218 2024-11-09T02:13:27,939 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218 after 1ms 2024-11-09T02:13:27,942 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(310): Open hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218 took 4ms 2024-11-09T02:13:27,946 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8aa375f3cce51bfb890245ebbdc0e977: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T02:13:27,947 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 88ba7a90c8e825401b464926d88f3b36: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T02:13:27,949 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 85da2d651446630bfba59fcb89f923a1: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T02:13:27,951 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for bbfdf2901c997d044e2c20f81e3a2a90: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T02:13:27,953 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for b3b3ba949727dd9291d8adc925fb09cb: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T02:13:27,955 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 3a2a5d6e6a65b43660fcb665b4ef5c83: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T02:13:27,957 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 8fe76b2cbbde488b7620457c1f89dcc9: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T02:13:27,959 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(352): Last flushed sequenceid for 195f23de0822a5bc7d5a1a2b4b16afdb: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-09T02:13:27,959 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218 so closing down 2024-11-09T02:13:27,959 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-09T02:13:27,959 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.OutputSink(145): 3 split writer threads finished 2024-11-09T02:13:27,959 INFO [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(425): Processed 8 edits across 0 Regions in 16 ms; skipped=8; WAL=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218, size=2.2 K, length=2237, corrupted=false, cancelled=false 2024-11-09T02:13:27,959 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] wal.WALSplitter(428): Completed split of hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218, journal: Splitting hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218, size=2.2 K (2237bytes) at 1731118407938Finishing writing output for hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218 so closing down at 1731118407959 (+21 ms)3 split writer threads finished at 1731118407959Processed 8 edits across 0 Regions in 16 ms; skipped=8; WAL=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218, size=2.2 K, length=2237, corrupted=false, cancelled=false at 1731118407959 2024-11-09T02:13:27,960 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218 2024-11-09T02:13:27,960 DEBUG [RS_LOG_REPLAY_OPS-regionserver/53f286d3af3b:0-0 {event_type=RS_LOG_REPLAY, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-09T02:13:27,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36011 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-09T02:13:27,964 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting/53f286d3af3b%2C33749%2C1731118402489.1731118404218 to hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/oldWALs 2024-11-09T02:13:27,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-09T02:13:27,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 in 195 msec 2024-11-09T02:13:27,968 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:27,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=57 2024-11-09T02:13:27,970 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 in 202 msec 2024-11-09T02:13:27,972 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,33749,1731118402489-splitting dir is empty, no logs to split. 2024-11-09T02:13:27,972 INFO [PEWorker-1 {}] master.SplitWALManager(105): 53f286d3af3b,33749,1731118402489 WAL count=0, meta=false 2024-11-09T02:13:27,972 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 53f286d3af3b,33749,1731118402489 WAL splitting is done? wals=0, meta=false 2024-11-09T02:13:27,975 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN}, {pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN}, {pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN}, {pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN}, {pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN}, {pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN}, {pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN}, {pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN}] 2024-11-09T02:13:27,977 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:27,979 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T02:13:27,984 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:27,985 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T02:13:27,986 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:27,987 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T02:13:27,989 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:27,990 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T02:13:27,991 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:27,991 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T02:13:27,992 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:27,993 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T02:13:27,994 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:27,995 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T02:13:27,996 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:27,997 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-09T02:13:28,129 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(204): Hosts are {53f286d3af3b=0} racks are {/default-rack=0} 2024-11-09T02:13:28,130 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-09T02:13:28,130 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-09T02:13:28,130 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-09T02:13:28,130 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-09T02:13:28,130 INFO [53f286d3af3b:36011 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-09T02:13:28,130 INFO [53f286d3af3b:36011 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-09T02:13:28,130 DEBUG [53f286d3af3b:36011 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-09T02:13:28,131 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=66 updating hbase:meta row=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,132 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=65 updating hbase:meta row=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,132 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,132 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, regionLocation=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,132 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 53f286d3af3b:33749 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:13:28,133 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,133 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,134 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1, error=java.net.ConnectException: Call to address=53f286d3af3b:33749 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 2024-11-09T02:13:28,134 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, regionLocation=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-09T02:13:28,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1 from cache 2024-11-09T02:13:28,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 53f286d3af3b:33749 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 2024-11-09T02:13:28,136 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 53f286d3af3b:33749 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-09T02:13:28,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=53f286d3af3b:33749 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 2024-11-09T02:13:28,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=53f286d3af3b:33749 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 2024-11-09T02:13:28,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=53f286d3af3b:33749 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 2024-11-09T02:13:28,136 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=53f286d3af3b:33749 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 2024-11-09T02:13:28,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=53f286d3af3b:33749 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 2024-11-09T02:13:28,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=53f286d3af3b:33749 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 2024-11-09T02:13:28,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,33749,1731118402489, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=53f286d3af3b:33749 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 2024-11-09T02:13:28,137 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 53f286d3af3b:33749 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 53f286d3af3b/172.17.0.3:33749 2024-11-09T02:13:28,242 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-09T02:13:28,242 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=53f286d3af3b,45941,1731118402318, seqNum=-1] 2024-11-09T02:13:28,242 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-09T02:13:28,244 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51781, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-09T02:13:28,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN because future has completed 2024-11-09T02:13:28,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:28,247 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN because future has completed 2024-11-09T02:13:28,248 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN because future has completed 2024-11-09T02:13:28,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN because future has completed 2024-11-09T02:13:28,251 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN because future has completed 2024-11-09T02:13:28,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:28,252 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN because future has completed 2024-11-09T02:13:28,253 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN because future has completed 2024-11-09T02:13:28,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:28,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN because future has completed 2024-11-09T02:13:28,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:28,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,46883,1731118402441}] 2024-11-09T02:13:28,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:28,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:28,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a2a5d6e6a65b43660fcb665b4ef5c83, server=53f286d3af3b,45941,1731118402318}] 2024-11-09T02:13:28,403 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:28,404 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => b3b3ba949727dd9291d8adc925fb09cb, NAME => 'test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-09T02:13:28,404 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:28,404 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:28,404 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:28,404 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:28,406 INFO [StoreOpener-b3b3ba949727dd9291d8adc925fb09cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:28,407 INFO [StoreOpener-b3b3ba949727dd9291d8adc925fb09cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b3b3ba949727dd9291d8adc925fb09cb columnFamilyName family 2024-11-09T02:13:28,408 DEBUG [StoreOpener-b3b3ba949727dd9291d8adc925fb09cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:28,408 INFO [StoreOpener-b3b3ba949727dd9291d8adc925fb09cb-1 {}] regionserver.HStore(327): Store=b3b3ba949727dd9291d8adc925fb09cb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:28,408 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:28,409 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:28,410 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:28,410 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 195f23de0822a5bc7d5a1a2b4b16afdb, NAME => 'test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-09T02:13:28,411 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:28,411 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:28,411 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:28,411 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:28,411 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:28,411 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:28,411 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:28,413 INFO [StoreOpener-195f23de0822a5bc7d5a1a2b4b16afdb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:28,414 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:28,414 INFO [StoreOpener-195f23de0822a5bc7d5a1a2b4b16afdb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 195f23de0822a5bc7d5a1a2b4b16afdb columnFamilyName family 2024-11-09T02:13:28,414 DEBUG [StoreOpener-195f23de0822a5bc7d5a1a2b4b16afdb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:28,414 INFO [StoreOpener-195f23de0822a5bc7d5a1a2b4b16afdb-1 {}] regionserver.HStore(327): Store=195f23de0822a5bc7d5a1a2b4b16afdb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:28,415 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:28,415 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened b3b3ba949727dd9291d8adc925fb09cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66675824, jitterRate=-0.006452798843383789}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:28,415 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:28,415 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for b3b3ba949727dd9291d8adc925fb09cb: Running coprocessor pre-open hook at 1731118408405Writing region info on filesystem at 1731118408405Initializing all the Stores at 1731118408406 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118408406Cleaning up temporary data from old regions at 1731118408411 (+5 ms)Running coprocessor post-open hooks at 1731118408415 (+4 ms)Region opened successfully at 1731118408415 2024-11-09T02:13:28,416 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:28,416 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb., pid=78, masterSystemTime=1731118408399 2024-11-09T02:13:28,417 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:28,418 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:28,418 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:28,419 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:28,419 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:28,419 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:28,419 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 8aa375f3cce51bfb890245ebbdc0e977, NAME => 'test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-09T02:13:28,420 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:28,420 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:28,420 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:28,420 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:28,421 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:28,422 INFO [StoreOpener-8aa375f3cce51bfb890245ebbdc0e977-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:28,422 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-09T02:13:28,422 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,423 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 195f23de0822a5bc7d5a1a2b4b16afdb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61890748, jitterRate=-0.07775598764419556}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:28,423 INFO [StoreOpener-8aa375f3cce51bfb890245ebbdc0e977-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8aa375f3cce51bfb890245ebbdc0e977 columnFamilyName family 2024-11-09T02:13:28,423 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:28,423 DEBUG [StoreOpener-8aa375f3cce51bfb890245ebbdc0e977-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:28,423 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 195f23de0822a5bc7d5a1a2b4b16afdb: Running coprocessor pre-open hook at 1731118408411Writing region info on filesystem at 1731118408411Initializing all the Stores at 1731118408412 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118408412Cleaning up temporary data from old regions at 1731118408418 (+6 ms)Running coprocessor post-open hooks at 1731118408423 (+5 ms)Region opened successfully at 1731118408423 2024-11-09T02:13:28,424 INFO [StoreOpener-8aa375f3cce51bfb890245ebbdc0e977-1 {}] regionserver.HStore(327): Store=8aa375f3cce51bfb890245ebbdc0e977/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:28,423 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,424 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:28,425 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb., pid=75, masterSystemTime=1731118408405 2024-11-09T02:13:28,425 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:28,427 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:28,428 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:28,428 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:28,428 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:28,428 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => 88ba7a90c8e825401b464926d88f3b36, NAME => 'test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-09T02:13:28,429 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:28,429 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:28,429 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:28,429 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:28,429 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:28,429 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:28,430 INFO [StoreOpener-88ba7a90c8e825401b464926d88f3b36-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:28,431 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:28,432 INFO [StoreOpener-88ba7a90c8e825401b464926d88f3b36-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88ba7a90c8e825401b464926d88f3b36 columnFamilyName family 2024-11-09T02:13:28,432 DEBUG [StoreOpener-88ba7a90c8e825401b464926d88f3b36-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:28,432 INFO [StoreOpener-88ba7a90c8e825401b464926d88f3b36-1 {}] regionserver.HStore(327): Store=88ba7a90c8e825401b464926d88f3b36/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:28,432 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:28,432 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 8aa375f3cce51bfb890245ebbdc0e977; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67260635, jitterRate=0.0022615641355514526}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:28,433 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:28,433 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 8aa375f3cce51bfb890245ebbdc0e977: Running coprocessor pre-open hook at 1731118408420Writing region info on filesystem at 1731118408420Initializing all the Stores at 1731118408421 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118408421Cleaning up temporary data from old regions at 1731118408429 (+8 ms)Running coprocessor post-open hooks at 1731118408433 (+4 ms)Region opened successfully at 1731118408433 2024-11-09T02:13:28,434 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:28,434 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977., pid=77, masterSystemTime=1731118408399 2024-11-09T02:13:28,435 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:28,437 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:28,437 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:28,435 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-09T02:13:28,438 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:28,438 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:28,438 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:28,438 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a2a5d6e6a65b43660fcb665b4ef5c83, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,438 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:28,438 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 3a2a5d6e6a65b43660fcb665b4ef5c83, NAME => 'test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-09T02:13:28,439 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:28,439 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a2a5d6e6a65b43660fcb665b4ef5c83, server=53f286d3af3b,45941,1731118402318 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,439 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:28,439 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:28,439 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:28,440 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:28,441 INFO [StoreOpener-3a2a5d6e6a65b43660fcb665b4ef5c83-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:28,441 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened 88ba7a90c8e825401b464926d88f3b36; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59971385, jitterRate=-0.10635672509670258}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:28,441 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:28,441 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for 88ba7a90c8e825401b464926d88f3b36: Running coprocessor pre-open hook at 1731118408429Writing region info on filesystem at 1731118408429Initializing all the Stores at 1731118408430 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118408430Cleaning up temporary data from old regions at 1731118408437 (+7 ms)Running coprocessor post-open hooks at 1731118408441 (+4 ms)Region opened successfully at 1731118408441 2024-11-09T02:13:28,442 INFO [StoreOpener-3a2a5d6e6a65b43660fcb665b4ef5c83-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3a2a5d6e6a65b43660fcb665b4ef5c83 columnFamilyName family 2024-11-09T02:13:28,442 DEBUG [StoreOpener-3a2a5d6e6a65b43660fcb665b4ef5c83-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:28,442 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36., pid=76, masterSystemTime=1731118408405 2024-11-09T02:13:28,443 INFO [StoreOpener-3a2a5d6e6a65b43660fcb665b4ef5c83-1 {}] regionserver.HStore(327): Store=3a2a5d6e6a65b43660fcb665b4ef5c83/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:28,443 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:28,444 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:28,446 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:28,446 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:28,446 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:28,446 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(132): Open test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:28,446 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7752): Opening region: {ENCODED => bbfdf2901c997d044e2c20f81e3a2a90, NAME => 'test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-09T02:13:28,446 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:28,446 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:28,446 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:28,446 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(898): Instantiated test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:28,446 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7794): checking encryption for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:28,447 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(7797): checking classloading for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:28,448 INFO [StoreOpener-bbfdf2901c997d044e2c20f81e3a2a90-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:28,448 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:28,449 INFO [StoreOpener-bbfdf2901c997d044e2c20f81e3a2a90-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bbfdf2901c997d044e2c20f81e3a2a90 columnFamilyName family 2024-11-09T02:13:28,449 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 3a2a5d6e6a65b43660fcb665b4ef5c83; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64306862, jitterRate=-0.04175308346748352}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:28,449 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:28,449 DEBUG [StoreOpener-bbfdf2901c997d044e2c20f81e3a2a90-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:28,450 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 3a2a5d6e6a65b43660fcb665b4ef5c83: Running coprocessor pre-open hook at 1731118408439Writing region info on filesystem at 1731118408439Initializing all the Stores at 1731118408440 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118408440Cleaning up temporary data from old regions at 1731118408446 (+6 ms)Running coprocessor post-open hooks at 1731118408449 (+3 ms)Region opened successfully at 1731118408449 2024-11-09T02:13:28,450 INFO [StoreOpener-bbfdf2901c997d044e2c20f81e3a2a90-1 {}] regionserver.HStore(327): Store=bbfdf2901c997d044e2c20f81e3a2a90/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:28,450 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1038): replaying wal for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:28,451 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83., pid=79, masterSystemTime=1731118408399 2024-11-09T02:13:28,451 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:28,453 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:28,454 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:28,454 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:28,454 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(132): Open test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:28,454 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1048): stopping wal replay for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:28,454 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1060): Cleaning up temporary data for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:28,454 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7752): Opening region: {ENCODED => 85da2d651446630bfba59fcb89f923a1, NAME => 'test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-09T02:13:28,454 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:28,455 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(898): Instantiated test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:28,455 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7794): checking encryption for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:28,455 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(7797): checking classloading for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:28,456 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1093): writing seq id for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:28,457 INFO [StoreOpener-85da2d651446630bfba59fcb89f923a1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:28,457 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1114): Opened bbfdf2901c997d044e2c20f81e3a2a90; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67858831, jitterRate=0.011175379157066345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:28,458 INFO [StoreOpener-85da2d651446630bfba59fcb89f923a1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 85da2d651446630bfba59fcb89f923a1 columnFamilyName family 2024-11-09T02:13:28,458 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:28,458 DEBUG [StoreOpener-85da2d651446630bfba59fcb89f923a1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:28,458 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegion(1006): Region open journal for bbfdf2901c997d044e2c20f81e3a2a90: Running coprocessor pre-open hook at 1731118408447Writing region info on filesystem at 1731118408447Initializing all the Stores at 1731118408448 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118408448Cleaning up temporary data from old regions at 1731118408454 (+6 ms)Running coprocessor post-open hooks at 1731118408458 (+4 ms)Region opened successfully at 1731118408458 2024-11-09T02:13:28,458 INFO [StoreOpener-85da2d651446630bfba59fcb89f923a1-1 {}] regionserver.HStore(327): Store=85da2d651446630bfba59fcb89f923a1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:28,458 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1038): replaying wal for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:28,459 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90., pid=73, masterSystemTime=1731118408405 2024-11-09T02:13:28,459 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:28,461 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:28,462 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:28,463 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=73}] handler.AssignRegionHandler(153): Opened test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:28,463 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(132): Open test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:28,463 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7752): Opening region: {ENCODED => 8fe76b2cbbde488b7620457c1f89dcc9, NAME => 'test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-09T02:13:28,463 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:28,464 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(898): Instantiated test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-09T02:13:28,464 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7794): checking encryption for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:28,464 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(7797): checking classloading for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:28,464 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1048): stopping wal replay for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:28,464 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1060): Cleaning up temporary data for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:28,466 INFO [StoreOpener-8fe76b2cbbde488b7620457c1f89dcc9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:28,467 INFO [StoreOpener-8fe76b2cbbde488b7620457c1f89dcc9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8fe76b2cbbde488b7620457c1f89dcc9 columnFamilyName family 2024-11-09T02:13:28,467 DEBUG [StoreOpener-8fe76b2cbbde488b7620457c1f89dcc9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-09T02:13:28,467 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1093): writing seq id for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:28,468 INFO [StoreOpener-8fe76b2cbbde488b7620457c1f89dcc9-1 {}] regionserver.HStore(327): Store=8fe76b2cbbde488b7620457c1f89dcc9/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-09T02:13:28,468 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1038): replaying wal for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:28,468 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1114): Opened 85da2d651446630bfba59fcb89f923a1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73486512, jitterRate=0.09503436088562012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:28,468 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:28,469 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:28,469 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegion(1006): Region open journal for 85da2d651446630bfba59fcb89f923a1: Running coprocessor pre-open hook at 1731118408455Writing region info on filesystem at 1731118408455Initializing all the Stores at 1731118408456 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118408456Cleaning up temporary data from old regions at 1731118408464 (+8 ms)Running coprocessor post-open hooks at 1731118408469 (+5 ms)Region opened successfully at 1731118408469 2024-11-09T02:13:28,470 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1., pid=72, masterSystemTime=1731118408399 2024-11-09T02:13:28,470 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:28,471 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1048): stopping wal replay for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:28,471 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1060): Cleaning up temporary data for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:28,474 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1093): writing seq id for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:28,475 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1114): Opened 8fe76b2cbbde488b7620457c1f89dcc9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63767539, jitterRate=-0.04978962242603302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-09T02:13:28,475 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:28,475 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegion(1006): Region open journal for 8fe76b2cbbde488b7620457c1f89dcc9: Running coprocessor pre-open hook at 1731118408464Writing region info on filesystem at 1731118408464Initializing all the Stores at 1731118408465 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731118408465Cleaning up temporary data from old regions at 1731118408471 (+6 ms)Running coprocessor post-open hooks at 1731118408475 (+4 ms)Region opened successfully at 1731118408475 2024-11-09T02:13:28,476 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:28,476 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9., pid=74, masterSystemTime=1731118408405 2024-11-09T02:13:28,476 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=72}] handler.AssignRegionHandler(153): Opened test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:28,479 DEBUG [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:28,479 INFO [RS_OPEN_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_OPEN_REGION, pid=74}] handler.AssignRegionHandler(153): Opened test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:28,512 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:28,512 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,512 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:28,513 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:28,513 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:28,513 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:28,513 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:28,514 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:28,515 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:28,520 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:28,520 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,520 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=65, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,521 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,522 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:28,524 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,528 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a2a5d6e6a65b43660fcb665b4ef5c83, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,529 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,529 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,529 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,529 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,530 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,530 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,530 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,530 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,530 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,530 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=65, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3a2a5d6e6a65b43660fcb665b4ef5c83, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,531 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-09T02:13:28,532 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:28,532 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:28,533 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:28,604 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,606 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,609 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:28,609 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:28,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:28,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:28,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:28,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN to state=OFFLINE, location=null, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9 to restore RIT 2024-11-09T02:13:28,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN to state=OFFLINE, location=null, table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb to restore RIT 2024-11-09T02:13:28,610 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN to state=OFFLINE, location=null, table=test, region=b3b3ba949727dd9291d8adc925fb09cb to restore RIT 2024-11-09T02:13:28,610 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118438610 2024-11-09T02:13:28,610 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:28,610 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:28,614 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:28,614 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,614 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,645 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,646 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,647 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,648 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,649 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,650 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,651 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,652 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,652 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:28,652 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118468652 2024-11-09T02:13:28,652 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118528652 2024-11-09T02:13:28,652 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 41 msec 2024-11-09T02:13:28,652 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:28,653 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,653 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:28,653 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:28,653 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:28,653 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:28,653 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:28,654 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:28,655 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,661 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,662 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,662 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,662 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,662 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,662 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,662 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,662 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:28,664 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:28,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:28,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:28,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,668 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,669 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,670 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,670 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=71, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,671 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-09T02:13:28,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:28,673 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:28,673 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:28,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,682 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,682 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:28,682 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:28,683 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:28,683 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:28,683 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:28,683 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN to state=OFFLINE, location=null, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9 to restore RIT 2024-11-09T02:13:28,683 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN to state=OFFLINE, location=null, table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb to restore RIT 2024-11-09T02:13:28,683 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN to state=OFFLINE, location=null, table=test, region=b3b3ba949727dd9291d8adc925fb09cb to restore RIT 2024-11-09T02:13:28,683 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118438683 2024-11-09T02:13:28,683 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:28,683 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,687 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:28,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:28,688 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,688 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,688 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,45941,1731118402318 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,697 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,698 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,699 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,700 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,701 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,702 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,702 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:28,702 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118468702 2024-11-09T02:13:28,702 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118528702 2024-11-09T02:13:28,702 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 19 msec 2024-11-09T02:13:28,703 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:28,703 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,703 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:28,703 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:28,703 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:28,703 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:28,703 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:28,703 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:28,704 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,710 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,711 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,711 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:28,712 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:28,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:28,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,715 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,716 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,717 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,717 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,718 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,718 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,718 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,718 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,718 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8aa375f3cce51bfb890245ebbdc0e977, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,718 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-09T02:13:28,720 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:28,720 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:28,720 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:28,728 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,729 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,731 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,732 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:28,732 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:28,732 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:28,733 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:28,733 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:28,733 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN to state=OFFLINE, location=null, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9 to restore RIT 2024-11-09T02:13:28,733 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN to state=OFFLINE, location=null, table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb to restore RIT 2024-11-09T02:13:28,733 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN to state=OFFLINE, location=null, table=test, region=b3b3ba949727dd9291d8adc925fb09cb to restore RIT 2024-11-09T02:13:28,733 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118438733 2024-11-09T02:13:28,733 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:28,733 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,736 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:28,737 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:28,737 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,738 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,46883,1731118402441 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,743 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,744 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,745 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,747 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,748 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,749 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,750 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,750 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:28,750 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118468750 2024-11-09T02:13:28,750 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118528750 2024-11-09T02:13:28,750 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 16 msec 2024-11-09T02:13:28,750 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:28,750 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,750 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:28,750 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:28,751 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:28,751 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:28,751 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:28,751 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:28,752 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:28,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:28,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,757 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,758 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:28,760 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:28,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:28,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,764 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,765 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,766 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,766 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,766 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,767 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,767 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,767 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,767 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,767 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,767 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,767 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,767 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,767 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,767 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88ba7a90c8e825401b464926d88f3b36, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,767 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-09T02:13:28,769 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:28,769 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:28,769 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:28,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,776 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,776 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,776 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,776 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,777 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:28,777 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:28,777 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:28,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:28,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:28,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN to state=OFFLINE, location=null, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9 to restore RIT 2024-11-09T02:13:28,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN to state=OFFLINE, location=null, table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb to restore RIT 2024-11-09T02:13:28,778 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN to state=OFFLINE, location=null, table=test, region=b3b3ba949727dd9291d8adc925fb09cb to restore RIT 2024-11-09T02:13:28,778 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118438778 2024-11-09T02:13:28,778 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:28,778 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,780 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:28,782 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:28,782 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,782 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,787 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,788 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,789 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,793 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,793 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:28,793 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118468793 2024-11-09T02:13:28,793 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118528793 2024-11-09T02:13:28,793 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 14 msec 2024-11-09T02:13:28,793 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:28,793 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,793 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:28,794 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:28,794 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:28,794 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:28,794 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:28,794 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:28,795 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:28,799 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,800 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,801 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,801 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:28,802 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:28,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:28,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:28,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,806 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,807 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,808 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,808 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,808 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,808 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,808 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,808 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,808 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,808 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,808 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,808 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,808 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,808 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,808 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,808 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,808 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,808 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,808 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,809 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,809 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,809 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,809 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 195f23de0822a5bc7d5a1a2b4b16afdb, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,809 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-09T02:13:28,810 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:28,811 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:28,812 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:28,836 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,837 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,857 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,864 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:28,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:28,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:28,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:28,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:28,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN to state=OFFLINE, location=null, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9 to restore RIT 2024-11-09T02:13:28,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN to state=OFFLINE, location=null, table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb to restore RIT 2024-11-09T02:13:28,865 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN to state=OFFLINE, location=null, table=test, region=b3b3ba949727dd9291d8adc925fb09cb to restore RIT 2024-11-09T02:13:28,869 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118438869 2024-11-09T02:13:28,869 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:28,870 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,872 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:28,877 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:28,876 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,877 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,877 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,877 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,878 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,879 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,880 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,881 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,881 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:28,882 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118468882 2024-11-09T02:13:28,882 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118528882 2024-11-09T02:13:28,882 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-09T02:13:28,882 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:28,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,882 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:28,882 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:28,882 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:28,882 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:28,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:28,883 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:28,885 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-09T02:13:28,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:28,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,887 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,888 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:28,889 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:28,891 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,892 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,893 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,893 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,893 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,893 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,894 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,894 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,894 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,894 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,894 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,894 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,894 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,894 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=74, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8fe76b2cbbde488b7620457c1f89dcc9, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,894 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-09T02:13:28,897 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:28,897 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:28,897 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:28,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:28,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,913 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:28,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:28,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:28,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:28,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:28,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:28,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN to state=OFFLINE, location=null, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9 to restore RIT 2024-11-09T02:13:28,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN to state=OFFLINE, location=null, table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb to restore RIT 2024-11-09T02:13:28,915 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN to state=OFFLINE, location=null, table=test, region=b3b3ba949727dd9291d8adc925fb09cb to restore RIT 2024-11-09T02:13:28,920 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118438920 2024-11-09T02:13:28,920 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:28,920 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,922 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:28,924 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:28,924 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,924 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,929 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,930 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,931 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,932 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:28,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,933 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:28,934 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:28,934 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118468934 2024-11-09T02:13:28,934 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118528934 2024-11-09T02:13:28,934 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-09T02:13:28,934 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:28,934 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:28,934 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:28,934 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:28,934 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:28,934 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:28,934 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:28,935 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:28,937 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-09T02:13:28,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:28,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=64, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,940 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:28,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:28,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:28,941 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:28,941 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:28,942 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:28,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:28,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:28,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,944 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,945 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:28,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:28,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:28,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:28,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:28,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:28,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:28,946 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:28,946 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:28,946 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,946 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=73, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure bbfdf2901c997d044e2c20f81e3a2a90, server=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:28,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-09T02:13:28,949 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:28,950 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:28,950 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:28,961 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,962 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:28,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741840_1016 (size=45) 2024-11-09T02:13:28,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741837_1013 (size=358) 2024-11-09T02:13:28,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741841_1017 (size=45) 2024-11-09T02:13:28,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741839_1015 (size=42) 2024-11-09T02:13:29,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,017 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,018 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:29,018 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:29,018 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:29,018 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:29,018 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:29,018 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN to state=OFFLINE, location=null, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9 to restore RIT 2024-11-09T02:13:29,018 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN to state=OFFLINE, location=null, table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb to restore RIT 2024-11-09T02:13:29,018 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN to state=OFFLINE, location=null, table=test, region=b3b3ba949727dd9291d8adc925fb09cb to restore RIT 2024-11-09T02:13:29,018 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439018 2024-11-09T02:13:29,019 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,019 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,020 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,021 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,021 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,021 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,025 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,026 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,027 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,028 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,028 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,028 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469028 2024-11-09T02:13:29,028 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529028 2024-11-09T02:13:29,028 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T02:13:29,028 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,028 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,028 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,028 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,029 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,029 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,029 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,029 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,030 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=66, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,032 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,033 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:29,035 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,036 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:29,036 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=72, ppid=66, state=RUNNABLE, hasLock=false; OpenRegionProcedure 85da2d651446630bfba59fcb89f923a1, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,036 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-09T02:13:29,037 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,037 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,037 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,082 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,082 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,082 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,082 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:29,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:29,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:29,083 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:29,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:29,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN to state=OFFLINE, location=null, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9 to restore RIT 2024-11-09T02:13:29,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN to state=OFFLINE, location=null, table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb to restore RIT 2024-11-09T02:13:29,084 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN to state=OFFLINE, location=null, table=test, region=b3b3ba949727dd9291d8adc925fb09cb to restore RIT 2024-11-09T02:13:29,084 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439084 2024-11-09T02:13:29,084 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,084 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,086 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,088 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,088 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:29,088 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,088 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,090 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,091 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,092 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,093 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,093 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,093 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469093 2024-11-09T02:13:29,093 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529093 2024-11-09T02:13:29,093 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-09T02:13:29,093 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,093 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,093 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,093 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,093 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,093 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,093 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,094 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,094 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,096 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,097 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,097 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,097 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,099 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:29,100 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,100 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN 2024-11-09T02:13:29,101 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-09T02:13:29,102 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,102 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,102 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,119 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,119 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,124 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,124 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:29,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:29,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:29,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:29,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:29,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN to state=OFFLINE, location=null, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9 to restore RIT 2024-11-09T02:13:29,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN to state=OFFLINE, location=null, table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb to restore RIT 2024-11-09T02:13:29,125 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=b3b3ba949727dd9291d8adc925fb09cb, ASSIGN to state=OFFLINE, location=null, table=test, region=b3b3ba949727dd9291d8adc925fb09cb to restore RIT 2024-11-09T02:13:29,125 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439125 2024-11-09T02:13:29,125 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,125 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,126 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,127 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,127 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:29,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,127 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,131 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,132 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,133 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,134 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,135 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,135 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,135 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469135 2024-11-09T02:13:29,135 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529135 2024-11-09T02:13:29,135 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T02:13:29,135 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,135 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,135 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,135 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,136 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,136 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,136 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,136 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,137 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,139 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,140 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,142 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,143 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:29,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:29,144 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,144 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN 2024-11-09T02:13:29,144 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-09T02:13:29,147 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,147 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,147 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,155 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,155 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,156 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:29,156 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:29,156 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:29,156 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:29,157 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:29,157 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN to state=OFFLINE, location=null, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9 to restore RIT 2024-11-09T02:13:29,157 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb, ASSIGN to state=OFFLINE, location=null, table=test, region=195f23de0822a5bc7d5a1a2b4b16afdb to restore RIT 2024-11-09T02:13:29,160 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439160 2024-11-09T02:13:29,160 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,160 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,161 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,162 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,162 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:29,162 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,163 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,171 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,171 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469171 2024-11-09T02:13:29,171 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529171 2024-11-09T02:13:29,171 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-09T02:13:29,171 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,171 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,171 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,171 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,172 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,172 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,172 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,172 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,173 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,175 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,176 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:29,177 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,178 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,178 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,178 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,178 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,178 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:29,178 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,178 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN 2024-11-09T02:13:29,178 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T02:13:29,179 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,179 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,180 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,187 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,187 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,187 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,187 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:29,187 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:29,188 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:29,188 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:29,188 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:29,188 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8fe76b2cbbde488b7620457c1f89dcc9, ASSIGN to state=OFFLINE, location=null, table=test, region=8fe76b2cbbde488b7620457c1f89dcc9 to restore RIT 2024-11-09T02:13:29,188 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439188 2024-11-09T02:13:29,188 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,188 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,189 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,190 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,190 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:29,190 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,190 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,193 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,194 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,196 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,196 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,196 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469196 2024-11-09T02:13:29,196 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529196 2024-11-09T02:13:29,196 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-09T02:13:29,196 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,196 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,196 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,196 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,197 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,197 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,197 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,197 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,198 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,199 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,200 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,200 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,201 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,202 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,202 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,202 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,202 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,202 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,202 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN 2024-11-09T02:13:29,202 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T02:13:29,203 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,203 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,203 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,215 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,215 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,215 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,216 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:29,217 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:29,217 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:29,217 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:29,217 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8aa375f3cce51bfb890245ebbdc0e977, ASSIGN to state=OFFLINE, location=null, table=test, region=8aa375f3cce51bfb890245ebbdc0e977 to restore RIT 2024-11-09T02:13:29,217 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439217 2024-11-09T02:13:29,217 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,217 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,219 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,220 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,220 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,221 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,226 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,226 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,226 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469226 2024-11-09T02:13:29,226 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529226 2024-11-09T02:13:29,226 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T02:13:29,226 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,227 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,227 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,227 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,227 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,227 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,227 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,228 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,229 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,230 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,230 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,231 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,232 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,233 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,233 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,233 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,233 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,233 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,233 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,233 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,233 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,233 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,233 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,233 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,233 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,233 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,234 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,234 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,234 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,234 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN 2024-11-09T02:13:29,234 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-09T02:13:29,235 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,235 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,235 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,247 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,247 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,247 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,247 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,247 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:29,248 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:29,248 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:29,248 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=88ba7a90c8e825401b464926d88f3b36, ASSIGN to state=OFFLINE, location=null, table=test, region=88ba7a90c8e825401b464926d88f3b36 to restore RIT 2024-11-09T02:13:29,248 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439248 2024-11-09T02:13:29,248 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,248 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,249 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,250 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,250 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,250 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,250 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,253 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,254 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,255 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,256 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,256 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469256 2024-11-09T02:13:29,256 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529256 2024-11-09T02:13:29,256 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-09T02:13:29,256 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,256 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,256 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,256 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,256 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,256 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,257 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,257 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,259 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,259 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,260 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,261 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,261 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,261 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,262 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,262 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN 2024-11-09T02:13:29,262 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T02:13:29,263 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,263 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,263 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,271 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,271 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,271 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:29,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:29,272 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=66, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=85da2d651446630bfba59fcb89f923a1, ASSIGN to state=OFFLINE, location=null, table=test, region=85da2d651446630bfba59fcb89f923a1 to restore RIT 2024-11-09T02:13:29,272 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439272 2024-11-09T02:13:29,272 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,272 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,274 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,275 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,275 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,275 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,275 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,278 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,279 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,280 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,281 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,281 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,281 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469281 2024-11-09T02:13:29,281 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529281 2024-11-09T02:13:29,281 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T02:13:29,281 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,282 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,282 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,282 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,282 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,282 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,282 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,282 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,283 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,285 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,287 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,287 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,288 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,288 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,288 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,288 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,288 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN 2024-11-09T02:13:29,288 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T02:13:29,289 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,289 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,289 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,332 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,332 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,332 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,333 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:29,333 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=65, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83, ASSIGN to state=OFFLINE, location=null, table=test, region=3a2a5d6e6a65b43660fcb665b4ef5c83 to restore RIT 2024-11-09T02:13:29,334 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439334 2024-11-09T02:13:29,334 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,334 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,335 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,336 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,336 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,336 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,337 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,340 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,341 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,342 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,343 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,343 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,343 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469343 2024-11-09T02:13:29,344 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529343 2024-11-09T02:13:29,344 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-09T02:13:29,344 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,344 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,344 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,344 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,344 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,344 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,344 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,345 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,345 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,348 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,348 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,349 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=57, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,351 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,352 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN 2024-11-09T02:13:29,352 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-09T02:13:29,353 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,353 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,353 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,375 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,376 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,376 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,376 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,376 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=64, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=bbfdf2901c997d044e2c20f81e3a2a90, ASSIGN to state=OFFLINE, location=null, table=test, region=bbfdf2901c997d044e2c20f81e3a2a90 to restore RIT 2024-11-09T02:13:29,376 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439376 2024-11-09T02:13:29,377 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,377 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,378 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,379 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,379 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,379 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,383 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,384 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,384 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,385 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469385 2024-11-09T02:13:29,385 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529385 2024-11-09T02:13:29,385 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-09T02:13:29,385 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,385 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,385 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,385 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,385 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,385 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,385 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,385 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,386 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,387 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=62, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,388 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,389 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=62, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,389 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-09T02:13:29,390 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,390 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,390 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,397 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,397 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,398 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439398 2024-11-09T02:13:29,398 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,398 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,400 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,402 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,402 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,402 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,407 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,407 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,407 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,407 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469407 2024-11-09T02:13:29,407 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529407 2024-11-09T02:13:29,407 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-09T02:13:29,407 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,407 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,407 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,407 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,407 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,408 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,408 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,410 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,410 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,411 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.1731118404218, worker=53f286d3af3b,46883,1731118402441 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,412 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-09T02:13:29,412 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,412 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,413 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,437 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,440 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,460 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,460 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,460 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,462 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439462 2024-11-09T02:13:29,462 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,462 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,463 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,464 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,464 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,464 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,467 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,467 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,467 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,467 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,468 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,469 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,470 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,470 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,470 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469470 2024-11-09T02:13:29,470 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529470 2024-11-09T02:13:29,470 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-09T02:13:29,470 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,470 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,470 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,470 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,471 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,471 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,471 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,471 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,472 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,474 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,474 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,474 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,474 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,474 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-09T02:13:29,474 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,475 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,476 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,476 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,476 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,476 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,476 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,476 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,476 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,476 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,476 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,476 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,476 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-09T02:13:29,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,477 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,477 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,534 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,535 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,536 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439536 2024-11-09T02:13:29,536 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,536 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,538 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,539 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,540 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,540 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,540 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,543 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,544 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,545 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,546 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,547 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,547 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,547 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469547 2024-11-09T02:13:29,547 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529547 2024-11-09T02:13:29,547 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-09T02:13:29,547 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,547 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,547 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,547 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,548 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,548 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,548 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,548 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,549 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,551 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=57, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-09T02:13:29,551 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,552 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,553 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-09T02:13:29,553 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,553 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,553 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,553 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=60, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,553 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-09T02:13:29,554 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,554 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,554 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,562 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,563 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36011 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,586 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,587 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439587 2024-11-09T02:13:29,587 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,587 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,588 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,589 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,589 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,590 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,592 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,593 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,595 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,595 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469595 2024-11-09T02:13:29,595 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529595 2024-11-09T02:13:29,595 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-09T02:13:29,595 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,596 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,596 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,596 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,596 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,596 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,597 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,599 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,599 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,599 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=59, ppid=58, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-09T02:13:29,599 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,600 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,601 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,601 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,601 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=59, ppid=58, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,601 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-09T02:13:29,602 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,602 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,602 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,608 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,608 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,609 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439609 2024-11-09T02:13:29,609 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,609 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,610 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,612 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,612 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,614 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,615 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,616 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,616 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,616 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469616 2024-11-09T02:13:29,616 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529616 2024-11-09T02:13:29,616 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 6 msec 2024-11-09T02:13:29,616 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,616 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,616 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,616 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,616 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,617 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,617 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,617 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,618 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,619 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,619 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=58, ppid=57, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-09T02:13:29,620 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,620 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,621 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,621 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,621 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,621 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=58, ppid=57, state=SUCCESS, hasLock=false; SplitWALProcedure 53f286d3af3b%2C33749%2C1731118402489.meta.1731118404764.meta, worker=53f286d3af3b,45941,1731118402318 didn't hold the lock before restarting, skip acquiring lock. 2024-11-09T02:13:29,621 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-09T02:13:29,622 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,622 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,622 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,629 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,629 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,629 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,630 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439630 2024-11-09T02:13:29,630 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,630 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,631 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,632 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-09T02:13:29,632 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,632 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,633 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,635 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,636 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,637 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,638 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,638 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469638 2024-11-09T02:13:29,638 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529638 2024-11-09T02:13:29,638 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-09T02:13:29,638 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-09T02:13:29,638 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,638 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:29,638 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:29,638 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-09T02:13:29,638 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-09T02:13:29,638 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-09T02:13:29,639 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-09T02:13:29,639 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-09T02:13:29,641 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=57, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-09T02:13:29,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-09T02:13:29,642 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-09T02:13:29,642 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true 2024-11-09T02:13:29,642 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-09T02:13:29,642 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 2 msec 2024-11-09T02:13:29,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=61 2024-11-09T02:13:29,643 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=53f286d3af3b,45941,1731118402318, table=hbase:meta, region=1588230740 2024-11-09T02:13:29,643 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 53f286d3af3b,45941,1731118402318, state=OPEN 2024-11-09T02:13:29,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,650 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-09T02:13:29,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,650 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-09T02:13:29,651 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731118439651 2024-11-09T02:13:29,651 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-09T02:13:29,651 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=57, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-09T02:13:29,652 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-09T02:13:29,653 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=57, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=78, ppid=71, state=RUNNABLE, hasLock=true; OpenRegionProcedure b3b3ba949727dd9291d8adc925fb09cb, server=53f286d3af3b,45941,1731118402318:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 53f286d3af3b,33749,1731118402489, splitWal=true, meta=true exec-time=2.9380 sec 2024-11-09T02:13:29,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bb1296b406bf2a79b39a6de9a4389fcc, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,656 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=195f23de0822a5bc7d5a1a2b4b16afdb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f28f1a8d6b5be48d6e6dca7a81da9c57, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f3f9d6837573933dea12bd60465775e, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5c24783f69139da02a463f7c484b941b, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8aa375f3cce51bfb890245ebbdc0e977, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=072df2da98728631979ee233c5a94c13, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6dd1285d59409a10928f32885afd0b51, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,657 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=555bb0a1c37bd4750a7abd799c897bc7, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8fe76b2cbbde488b7620457c1f89dcc9, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=a5a881b27904120149acda46e3d36994, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ad814e81d5536ce802390dd5602e78a1, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=b3b3ba949727dd9291d8adc925fb09cb, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=244d470b9561d8921c93364fa1596bac, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=88ba7a90c8e825401b464926d88f3b36, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=7cb62548224e02c76c614c52af2aadca, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=923e0410f16fe611d9240651fc8bf39e, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=3a2a5d6e6a65b43660fcb665b4ef5c83, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c5c32774defefb99b01b165b020a41fa, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=194b5628d516a374d3336a52e639da04, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=87e2ba48090bed7cebebc5b429271688, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=bbfdf2901c997d044e2c20f81e3a2a90, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,658 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f372b843955cc5a3ace82b165e5d33b0, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=e33ccfa49d4634ee727e7cd03b929a72, regionState=OPEN, lastHost=53f286d3af3b,46883,1731118402441, regionLocation=53f286d3af3b,46883,1731118402441, openSeqNum=2 2024-11-09T02:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=85da2d651446630bfba59fcb89f923a1, regionState=OPENING, lastHost=53f286d3af3b,33749,1731118402489, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,659 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6e77447186c114b3577a6f77b1ffe798, regionState=OPEN, lastHost=53f286d3af3b,45941,1731118402318, regionLocation=53f286d3af3b,45941,1731118402318, openSeqNum=2 2024-11-09T02:13:29,659 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-09T02:13:29,659 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731118469659 2024-11-09T02:13:29,659 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731118529659 2024-11-09T02:13:29,659 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-09T02:13:29,674 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=348 (was 374), OpenFileDescriptor=604 (was 619), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 319), ProcessCount=11 (was 11), AvailableMemoryMB=2834 (was 2899) 2024-11-09T02:13:29,674 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-09T02:13:29,675 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T02:13:29,675 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:13:29,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:13:29,675 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:13:29,675 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-09T02:13:29,675 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-09T02:13:29,676 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=2048927636, stopped=false 2024-11-09T02:13:29,676 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=53f286d3af3b,36011,1731118401306 2024-11-09T02:13:29,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T02:13:29,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T02:13:29,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:29,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:29,681 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:29,681 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-09T02:13:29,682 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:29,682 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-09T02:13:29,682 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:13:29,682 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:13:29,682 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '53f286d3af3b,45941,1731118402318' ***** 2024-11-09T02:13:29,682 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T02:13:29,682 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '53f286d3af3b,46883,1731118402441' ***** 2024-11-09T02:13:29,682 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-09T02:13:29,683 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T02:13:29,683 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:13:29,683 INFO [RS:0;53f286d3af3b:45941 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T02:13:29,683 INFO [RS:0;53f286d3af3b:45941 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T02:13:29,683 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T02:13:29,683 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-09T02:13:29,683 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for a5a881b27904120149acda46e3d36994 2024-11-09T02:13:29,683 INFO [RS:1;53f286d3af3b:46883 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-09T02:13:29,683 INFO [RS:1;53f286d3af3b:46883 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-09T02:13:29,683 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for c5c32774defefb99b01b165b020a41fa 2024-11-09T02:13:29,683 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-09T02:13:29,683 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for 923e0410f16fe611d9240651fc8bf39e 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for 5c24783f69139da02a463f7c484b941b 2024-11-09T02:13:29,684 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for 8aa375f3cce51bfb890245ebbdc0e977 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for bb1296b406bf2a79b39a6de9a4389fcc 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for 6dd1285d59409a10928f32885afd0b51 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for 85da2d651446630bfba59fcb89f923a1 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for 195f23de0822a5bc7d5a1a2b4b16afdb 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for 6e77447186c114b3577a6f77b1ffe798 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for 87e2ba48090bed7cebebc5b429271688 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for 244d470b9561d8921c93364fa1596bac 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for e33ccfa49d4634ee727e7cd03b929a72 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for b3b3ba949727dd9291d8adc925fb09cb 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for 194b5628d516a374d3336a52e639da04 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for bbfdf2901c997d044e2c20f81e3a2a90 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for 3a2a5d6e6a65b43660fcb665b4ef5c83 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for 072df2da98728631979ee233c5a94c13 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for 555bb0a1c37bd4750a7abd799c897bc7 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(959): stopping server 53f286d3af3b,45941,1731118402318 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for 88ba7a90c8e825401b464926d88f3b36 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for 8fe76b2cbbde488b7620457c1f89dcc9 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for 7cb62548224e02c76c614c52af2aadca 2024-11-09T02:13:29,684 INFO [RS:0;53f286d3af3b:45941 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;53f286d3af3b:45941. 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for ad814e81d5536ce802390dd5602e78a1 2024-11-09T02:13:29,684 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(3091): Received CLOSE for 5f3f9d6837573933dea12bd60465775e 2024-11-09T02:13:29,684 DEBUG [RS:0;53f286d3af3b:45941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:13:29,685 DEBUG [RS:0;53f286d3af3b:45941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:13:29,685 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(959): stopping server 53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,685 INFO [RS:1;53f286d3af3b:46883 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T02:13:29,685 INFO [RS:1;53f286d3af3b:46883 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;53f286d3af3b:46883. 2024-11-09T02:13:29,685 DEBUG [RS:1;53f286d3af3b:46883 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-09T02:13:29,685 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T02:13:29,685 DEBUG [RS:1;53f286d3af3b:46883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:13:29,685 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing a5a881b27904120149acda46e3d36994, disabling compactions & flushes 2024-11-09T02:13:29,685 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T02:13:29,685 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T02:13:29,685 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:29,685 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-09T02:13:29,685 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-09T02:13:29,685 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:29,685 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. after waiting 0 ms 2024-11-09T02:13:29,685 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:29,685 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-09T02:13:29,686 DEBUG [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(1351): Waiting on 072df2da98728631979ee233c5a94c13, 195f23de0822a5bc7d5a1a2b4b16afdb, 555bb0a1c37bd4750a7abd799c897bc7, 5f3f9d6837573933dea12bd60465775e, 7cb62548224e02c76c614c52af2aadca, 87e2ba48090bed7cebebc5b429271688, 88ba7a90c8e825401b464926d88f3b36, 8fe76b2cbbde488b7620457c1f89dcc9, ad814e81d5536ce802390dd5602e78a1, bbfdf2901c997d044e2c20f81e3a2a90, c5c32774defefb99b01b165b020a41fa, e33ccfa49d4634ee727e7cd03b929a72, f28f1a8d6b5be48d6e6dca7a81da9c57 2024-11-09T02:13:29,686 DEBUG [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 194b5628d516a374d3336a52e639da04, 244d470b9561d8921c93364fa1596bac, 3a2a5d6e6a65b43660fcb665b4ef5c83, 5c24783f69139da02a463f7c484b941b, 6dd1285d59409a10928f32885afd0b51, 6e77447186c114b3577a6f77b1ffe798, 85da2d651446630bfba59fcb89f923a1, 8aa375f3cce51bfb890245ebbdc0e977, 923e0410f16fe611d9240651fc8bf39e, a5a881b27904120149acda46e3d36994, b3b3ba949727dd9291d8adc925fb09cb, bb1296b406bf2a79b39a6de9a4389fcc, f372b843955cc5a3ace82b165e5d33b0 2024-11-09T02:13:29,686 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c5c32774defefb99b01b165b020a41fa, disabling compactions & flushes 2024-11-09T02:13:29,686 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-09T02:13:29,686 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:29,686 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-09T02:13:29,686 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:29,686 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-09T02:13:29,686 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. after waiting 0 ms 2024-11-09T02:13:29,686 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-09T02:13:29,686 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:29,686 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-09T02:13:29,687 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.66 KB heapSize=5.44 KB 2024-11-09T02:13:29,692 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/a5a881b27904120149acda46e3d36994/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,692 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/c5c32774defefb99b01b165b020a41fa/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,694 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:29,694 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:29,694 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c5c32774defefb99b01b165b020a41fa: Waiting for close lock at 1731118409686Running coprocessor pre-close hooks at 1731118409686Disabling compacts and flushes for region at 1731118409686Disabling writes for close at 1731118409686Writing region close event to WAL at 1731118409687 (+1 ms)Running coprocessor post-close hooks at 1731118409693 (+6 ms)Closed at 1731118409694 (+1 ms) 2024-11-09T02:13:29,694 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for a5a881b27904120149acda46e3d36994: Waiting for close lock at 1731118409684Running coprocessor pre-close hooks at 1731118409685 (+1 ms)Disabling compacts and flushes for region at 1731118409685Disabling writes for close at 1731118409685Writing region close event to WAL at 1731118409686 (+1 ms)Running coprocessor post-close hooks at 1731118409693 (+7 ms)Closed at 1731118409694 (+1 ms) 2024-11-09T02:13:29,695 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1731118405247.c5c32774defefb99b01b165b020a41fa. 2024-11-09T02:13:29,695 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1731118405247.a5a881b27904120149acda46e3d36994. 2024-11-09T02:13:29,695 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f28f1a8d6b5be48d6e6dca7a81da9c57, disabling compactions & flushes 2024-11-09T02:13:29,695 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f372b843955cc5a3ace82b165e5d33b0, disabling compactions & flushes 2024-11-09T02:13:29,695 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:29,695 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:29,695 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:29,695 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:29,695 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. after waiting 0 ms 2024-11-09T02:13:29,695 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. after waiting 0 ms 2024-11-09T02:13:29,695 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:29,696 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:29,701 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/f28f1a8d6b5be48d6e6dca7a81da9c57/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,702 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/f372b843955cc5a3ace82b165e5d33b0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,703 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/info/b81227f35fd9439a99f7b9b0743331a0 is 138, key is test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb./info:regioninfo/1731118408134/Put/seqid=0 2024-11-09T02:13:29,704 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:29,704 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f28f1a8d6b5be48d6e6dca7a81da9c57: Waiting for close lock at 1731118409695Running coprocessor pre-close hooks at 1731118409695Disabling compacts and flushes for region at 1731118409695Disabling writes for close at 1731118409695Writing region close event to WAL at 1731118409696 (+1 ms)Running coprocessor post-close hooks at 1731118409704 (+8 ms)Closed at 1731118409704 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f372b843955cc5a3ace82b165e5d33b0: Waiting for close lock at 1731118409695Running coprocessor pre-close hooks at 1731118409695Disabling compacts and flushes for region at 1731118409695Disabling writes for close at 1731118409695Writing region close event to WAL at 1731118409696 (+1 ms)Running coprocessor post-close hooks at 1731118409703 (+7 ms)Closed at 1731118409704 (+1 ms) 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1731118405247.f372b843955cc5a3ace82b165e5d33b0. 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1731118405247.f28f1a8d6b5be48d6e6dca7a81da9c57. 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 923e0410f16fe611d9240651fc8bf39e, disabling compactions & flushes 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 195f23de0822a5bc7d5a1a2b4b16afdb, disabling compactions & flushes 2024-11-09T02:13:29,704 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:29,704 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. after waiting 0 ms 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. after waiting 0 ms 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:29,704 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:29,717 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/923e0410f16fe611d9240651fc8bf39e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,718 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:29,718 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 923e0410f16fe611d9240651fc8bf39e: Waiting for close lock at 1731118409704Running coprocessor pre-close hooks at 1731118409704Disabling compacts and flushes for region at 1731118409704Disabling writes for close at 1731118409704Writing region close event to WAL at 1731118409706 (+2 ms)Running coprocessor post-close hooks at 1731118409718 (+12 ms)Closed at 1731118409718 2024-11-09T02:13:29,718 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1731118405247.923e0410f16fe611d9240651fc8bf39e. 2024-11-09T02:13:29,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741869_1045 (size=8195) 2024-11-09T02:13:29,718 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5c24783f69139da02a463f7c484b941b, disabling compactions & flushes 2024-11-09T02:13:29,718 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:29,718 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:29,718 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. after waiting 0 ms 2024-11-09T02:13:29,718 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:29,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741869_1045 (size=8195) 2024-11-09T02:13:29,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741869_1045 (size=8195) 2024-11-09T02:13:29,719 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.66 KB at sequenceid=71 (bloomFilter=true), to=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/info/b81227f35fd9439a99f7b9b0743331a0 2024-11-09T02:13:29,720 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/195f23de0822a5bc7d5a1a2b4b16afdb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,721 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:29,721 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 195f23de0822a5bc7d5a1a2b4b16afdb: Waiting for close lock at 1731118409704Running coprocessor pre-close hooks at 1731118409704Disabling compacts and flushes for region at 1731118409704Disabling writes for close at 1731118409704Writing region close event to WAL at 1731118409705 (+1 ms)Running coprocessor post-close hooks at 1731118409721 (+16 ms)Closed at 1731118409721 2024-11-09T02:13:29,721 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1731118405247.195f23de0822a5bc7d5a1a2b4b16afdb. 2024-11-09T02:13:29,722 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 87e2ba48090bed7cebebc5b429271688, disabling compactions & flushes 2024-11-09T02:13:29,722 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:29,722 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:29,722 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. after waiting 0 ms 2024-11-09T02:13:29,722 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:29,726 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/5c24783f69139da02a463f7c484b941b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,727 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:29,727 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5c24783f69139da02a463f7c484b941b: Waiting for close lock at 1731118409718Running coprocessor pre-close hooks at 1731118409718Disabling compacts and flushes for region at 1731118409718Disabling writes for close at 1731118409718Writing region close event to WAL at 1731118409720 (+2 ms)Running coprocessor post-close hooks at 1731118409727 (+7 ms)Closed at 1731118409727 2024-11-09T02:13:29,727 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1731118405247.5c24783f69139da02a463f7c484b941b. 2024-11-09T02:13:29,727 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8aa375f3cce51bfb890245ebbdc0e977, disabling compactions & flushes 2024-11-09T02:13:29,727 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:29,727 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:29,727 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. after waiting 0 ms 2024-11-09T02:13:29,727 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:29,727 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/87e2ba48090bed7cebebc5b429271688/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,728 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:29,728 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 87e2ba48090bed7cebebc5b429271688: Waiting for close lock at 1731118409722Running coprocessor pre-close hooks at 1731118409722Disabling compacts and flushes for region at 1731118409722Disabling writes for close at 1731118409722Writing region close event to WAL at 1731118409722Running coprocessor post-close hooks at 1731118409728 (+6 ms)Closed at 1731118409728 2024-11-09T02:13:29,729 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1731118405247.87e2ba48090bed7cebebc5b429271688. 2024-11-09T02:13:29,729 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing e33ccfa49d4634ee727e7cd03b929a72, disabling compactions & flushes 2024-11-09T02:13:29,729 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:29,729 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:29,729 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. after waiting 0 ms 2024-11-09T02:13:29,729 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:29,730 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/.tmp/info/b81227f35fd9439a99f7b9b0743331a0 as hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/info/b81227f35fd9439a99f7b9b0743331a0 2024-11-09T02:13:29,733 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8aa375f3cce51bfb890245ebbdc0e977/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,734 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:29,734 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/e33ccfa49d4634ee727e7cd03b929a72/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,734 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8aa375f3cce51bfb890245ebbdc0e977: Waiting for close lock at 1731118409727Running coprocessor pre-close hooks at 1731118409727Disabling compacts and flushes for region at 1731118409727Disabling writes for close at 1731118409727Writing region close event to WAL at 1731118409728 (+1 ms)Running coprocessor post-close hooks at 1731118409734 (+6 ms)Closed at 1731118409734 2024-11-09T02:13:29,734 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1731118405247.8aa375f3cce51bfb890245ebbdc0e977. 2024-11-09T02:13:29,734 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bb1296b406bf2a79b39a6de9a4389fcc, disabling compactions & flushes 2024-11-09T02:13:29,734 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:29,734 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:29,734 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. after waiting 0 ms 2024-11-09T02:13:29,734 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:29,734 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:29,735 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for e33ccfa49d4634ee727e7cd03b929a72: Waiting for close lock at 1731118409729Running coprocessor pre-close hooks at 1731118409729Disabling compacts and flushes for region at 1731118409729Disabling writes for close at 1731118409729Writing region close event to WAL at 1731118409729Running coprocessor post-close hooks at 1731118409734 (+5 ms)Closed at 1731118409734 2024-11-09T02:13:29,735 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1731118405247.e33ccfa49d4634ee727e7cd03b929a72. 2024-11-09T02:13:29,735 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bbfdf2901c997d044e2c20f81e3a2a90, disabling compactions & flushes 2024-11-09T02:13:29,735 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:29,735 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:29,735 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. after waiting 0 ms 2024-11-09T02:13:29,735 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:29,739 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/bb1296b406bf2a79b39a6de9a4389fcc/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,740 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/info/b81227f35fd9439a99f7b9b0743331a0, entries=24, sequenceid=71, filesize=8.0 K 2024-11-09T02:13:29,741 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:29,741 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bb1296b406bf2a79b39a6de9a4389fcc: Waiting for close lock at 1731118409734Running coprocessor pre-close hooks at 1731118409734Disabling compacts and flushes for region at 1731118409734Disabling writes for close at 1731118409734Writing region close event to WAL at 1731118409735 (+1 ms)Running coprocessor post-close hooks at 1731118409740 (+5 ms)Closed at 1731118409741 (+1 ms) 2024-11-09T02:13:29,741 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1731118405247.bb1296b406bf2a79b39a6de9a4389fcc. 2024-11-09T02:13:29,741 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6dd1285d59409a10928f32885afd0b51, disabling compactions & flushes 2024-11-09T02:13:29,741 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 55ms, sequenceid=71, compaction requested=false 2024-11-09T02:13:29,741 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:29,741 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:29,741 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. after waiting 0 ms 2024-11-09T02:13:29,741 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:29,745 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/bbfdf2901c997d044e2c20f81e3a2a90/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,746 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:29,746 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bbfdf2901c997d044e2c20f81e3a2a90: Waiting for close lock at 1731118409735Running coprocessor pre-close hooks at 1731118409735Disabling compacts and flushes for region at 1731118409735Disabling writes for close at 1731118409735Writing region close event to WAL at 1731118409738 (+3 ms)Running coprocessor post-close hooks at 1731118409746 (+8 ms)Closed at 1731118409746 2024-11-09T02:13:29,746 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1731118405247.bbfdf2901c997d044e2c20f81e3a2a90. 2024-11-09T02:13:29,746 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 072df2da98728631979ee233c5a94c13, disabling compactions & flushes 2024-11-09T02:13:29,746 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:29,746 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:29,747 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. after waiting 0 ms 2024-11-09T02:13:29,747 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:29,747 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/6dd1285d59409a10928f32885afd0b51/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,748 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:29,748 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6dd1285d59409a10928f32885afd0b51: Waiting for close lock at 1731118409741Running coprocessor pre-close hooks at 1731118409741Disabling compacts and flushes for region at 1731118409741Disabling writes for close at 1731118409741Writing region close event to WAL at 1731118409742 (+1 ms)Running coprocessor post-close hooks at 1731118409748 (+6 ms)Closed at 1731118409748 2024-11-09T02:13:29,749 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1731118405247.6dd1285d59409a10928f32885afd0b51. 2024-11-09T02:13:29,749 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 85da2d651446630bfba59fcb89f923a1, disabling compactions & flushes 2024-11-09T02:13:29,749 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:29,749 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:29,749 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. after waiting 0 ms 2024-11-09T02:13:29,749 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:29,751 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/hbase/meta/1588230740/recovered.edits/74.seqid, newMaxSeqId=74, maxSeqId=60 2024-11-09T02:13:29,752 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/072df2da98728631979ee233c5a94c13/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,752 INFO [regionserver/53f286d3af3b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T02:13:29,752 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-09T02:13:29,752 INFO [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-09T02:13:29,752 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731118409686Running coprocessor pre-close hooks at 1731118409686Disabling compacts and flushes for region at 1731118409686Disabling writes for close at 1731118409686Obtaining lock to block concurrent updates at 1731118409687 (+1 ms)Preparing flush snapshotting stores in 1588230740 at 1731118409687Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2720, getHeapSize=5504, getOffHeapSize=0, getCellsCount=24 at 1731118409696 (+9 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731118409698 (+2 ms)Flushing 1588230740/info: creating writer at 1731118409698Flushing 1588230740/info: appending metadata at 1731118409702 (+4 ms)Flushing 1588230740/info: closing flushed file at 1731118409702Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@42c58cc: reopening flushed file at 1731118409728 (+26 ms)Finished flush of dataSize ~2.66 KB/2720, heapSize ~4.67 KB/4784, currentSize=0 B/0 for 1588230740 in 55ms, sequenceid=71, compaction requested=false at 1731118409741 (+13 ms)Writing region close event to WAL at 1731118409746 (+5 ms)Running coprocessor post-close hooks at 1731118409752 (+6 ms)Closed at 1731118409752 2024-11-09T02:13:29,752 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:29,752 DEBUG [RS_CLOSE_META-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-09T02:13:29,752 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 072df2da98728631979ee233c5a94c13: Waiting for close lock at 1731118409746Running coprocessor pre-close hooks at 1731118409746Disabling compacts and flushes for region at 1731118409746Disabling writes for close at 1731118409747 (+1 ms)Writing region close event to WAL at 1731118409747Running coprocessor post-close hooks at 1731118409752 (+5 ms)Closed at 1731118409752 2024-11-09T02:13:29,752 INFO [regionserver/53f286d3af3b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T02:13:29,752 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1731118405247.072df2da98728631979ee233c5a94c13. 2024-11-09T02:13:29,753 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 555bb0a1c37bd4750a7abd799c897bc7, disabling compactions & flushes 2024-11-09T02:13:29,753 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:29,753 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:29,753 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. after waiting 0 ms 2024-11-09T02:13:29,753 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:29,754 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/85da2d651446630bfba59fcb89f923a1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,754 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:29,754 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 85da2d651446630bfba59fcb89f923a1: Waiting for close lock at 1731118409749Running coprocessor pre-close hooks at 1731118409749Disabling compacts and flushes for region at 1731118409749Disabling writes for close at 1731118409749Writing region close event to WAL at 1731118409750 (+1 ms)Running coprocessor post-close hooks at 1731118409754 (+4 ms)Closed at 1731118409754 2024-11-09T02:13:29,755 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1731118405247.85da2d651446630bfba59fcb89f923a1. 2024-11-09T02:13:29,755 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6e77447186c114b3577a6f77b1ffe798, disabling compactions & flushes 2024-11-09T02:13:29,755 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:29,755 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:29,755 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. after waiting 0 ms 2024-11-09T02:13:29,755 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:29,757 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/555bb0a1c37bd4750a7abd799c897bc7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,758 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:29,758 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 555bb0a1c37bd4750a7abd799c897bc7: Waiting for close lock at 1731118409753Running coprocessor pre-close hooks at 1731118409753Disabling compacts and flushes for region at 1731118409753Disabling writes for close at 1731118409753Writing region close event to WAL at 1731118409753Running coprocessor post-close hooks at 1731118409758 (+5 ms)Closed at 1731118409758 2024-11-09T02:13:29,758 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1731118405247.555bb0a1c37bd4750a7abd799c897bc7. 2024-11-09T02:13:29,758 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 88ba7a90c8e825401b464926d88f3b36, disabling compactions & flushes 2024-11-09T02:13:29,758 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:29,759 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:29,759 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. after waiting 0 ms 2024-11-09T02:13:29,759 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:29,759 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/6e77447186c114b3577a6f77b1ffe798/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,760 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:29,760 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6e77447186c114b3577a6f77b1ffe798: Waiting for close lock at 1731118409755Running coprocessor pre-close hooks at 1731118409755Disabling compacts and flushes for region at 1731118409755Disabling writes for close at 1731118409755Writing region close event to WAL at 1731118409755Running coprocessor post-close hooks at 1731118409760 (+5 ms)Closed at 1731118409760 2024-11-09T02:13:29,760 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1731118405247.6e77447186c114b3577a6f77b1ffe798. 2024-11-09T02:13:29,760 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 244d470b9561d8921c93364fa1596bac, disabling compactions & flushes 2024-11-09T02:13:29,760 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:29,760 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:29,760 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. after waiting 0 ms 2024-11-09T02:13:29,761 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:29,763 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/88ba7a90c8e825401b464926d88f3b36/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,764 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:29,764 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 88ba7a90c8e825401b464926d88f3b36: Waiting for close lock at 1731118409758Running coprocessor pre-close hooks at 1731118409758Disabling compacts and flushes for region at 1731118409758Disabling writes for close at 1731118409759 (+1 ms)Writing region close event to WAL at 1731118409759Running coprocessor post-close hooks at 1731118409764 (+5 ms)Closed at 1731118409764 2024-11-09T02:13:29,764 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1731118405247.88ba7a90c8e825401b464926d88f3b36. 2024-11-09T02:13:29,764 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8fe76b2cbbde488b7620457c1f89dcc9, disabling compactions & flushes 2024-11-09T02:13:29,764 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:29,764 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:29,764 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/244d470b9561d8921c93364fa1596bac/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,764 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. after waiting 0 ms 2024-11-09T02:13:29,764 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:29,765 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:29,765 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 244d470b9561d8921c93364fa1596bac: Waiting for close lock at 1731118409760Running coprocessor pre-close hooks at 1731118409760Disabling compacts and flushes for region at 1731118409760Disabling writes for close at 1731118409760Writing region close event to WAL at 1731118409761 (+1 ms)Running coprocessor post-close hooks at 1731118409765 (+4 ms)Closed at 1731118409765 2024-11-09T02:13:29,765 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1731118405247.244d470b9561d8921c93364fa1596bac. 2024-11-09T02:13:29,765 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing b3b3ba949727dd9291d8adc925fb09cb, disabling compactions & flushes 2024-11-09T02:13:29,765 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:29,765 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:29,765 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. after waiting 0 ms 2024-11-09T02:13:29,765 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:29,768 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/8fe76b2cbbde488b7620457c1f89dcc9/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,769 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:29,769 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8fe76b2cbbde488b7620457c1f89dcc9: Waiting for close lock at 1731118409764Running coprocessor pre-close hooks at 1731118409764Disabling compacts and flushes for region at 1731118409764Disabling writes for close at 1731118409764Writing region close event to WAL at 1731118409765 (+1 ms)Running coprocessor post-close hooks at 1731118409769 (+4 ms)Closed at 1731118409769 2024-11-09T02:13:29,769 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1731118405247.8fe76b2cbbde488b7620457c1f89dcc9. 2024-11-09T02:13:29,770 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 7cb62548224e02c76c614c52af2aadca, disabling compactions & flushes 2024-11-09T02:13:29,770 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:29,770 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:29,770 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. after waiting 0 ms 2024-11-09T02:13:29,770 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:29,770 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/b3b3ba949727dd9291d8adc925fb09cb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,771 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:29,771 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for b3b3ba949727dd9291d8adc925fb09cb: Waiting for close lock at 1731118409765Running coprocessor pre-close hooks at 1731118409765Disabling compacts and flushes for region at 1731118409765Disabling writes for close at 1731118409765Writing region close event to WAL at 1731118409766 (+1 ms)Running coprocessor post-close hooks at 1731118409771 (+5 ms)Closed at 1731118409771 2024-11-09T02:13:29,771 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1731118405247.b3b3ba949727dd9291d8adc925fb09cb. 2024-11-09T02:13:29,772 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 194b5628d516a374d3336a52e639da04, disabling compactions & flushes 2024-11-09T02:13:29,772 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:29,772 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:29,772 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. after waiting 0 ms 2024-11-09T02:13:29,772 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:29,775 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/7cb62548224e02c76c614c52af2aadca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,775 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:29,775 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 7cb62548224e02c76c614c52af2aadca: Waiting for close lock at 1731118409770Running coprocessor pre-close hooks at 1731118409770Disabling compacts and flushes for region at 1731118409770Disabling writes for close at 1731118409770Writing region close event to WAL at 1731118409770Running coprocessor post-close hooks at 1731118409775 (+5 ms)Closed at 1731118409775 2024-11-09T02:13:29,776 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1731118405247.7cb62548224e02c76c614c52af2aadca. 2024-11-09T02:13:29,776 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ad814e81d5536ce802390dd5602e78a1, disabling compactions & flushes 2024-11-09T02:13:29,776 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:29,776 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:29,776 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. after waiting 0 ms 2024-11-09T02:13:29,776 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:29,777 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/194b5628d516a374d3336a52e639da04/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,778 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:29,778 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 194b5628d516a374d3336a52e639da04: Waiting for close lock at 1731118409772Running coprocessor pre-close hooks at 1731118409772Disabling compacts and flushes for region at 1731118409772Disabling writes for close at 1731118409772Writing region close event to WAL at 1731118409772Running coprocessor post-close hooks at 1731118409778 (+6 ms)Closed at 1731118409778 2024-11-09T02:13:29,778 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1731118405247.194b5628d516a374d3336a52e639da04. 2024-11-09T02:13:29,778 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3a2a5d6e6a65b43660fcb665b4ef5c83, disabling compactions & flushes 2024-11-09T02:13:29,778 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:29,778 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:29,778 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. after waiting 0 ms 2024-11-09T02:13:29,778 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:29,781 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/ad814e81d5536ce802390dd5602e78a1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,782 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:29,782 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ad814e81d5536ce802390dd5602e78a1: Waiting for close lock at 1731118409776Running coprocessor pre-close hooks at 1731118409776Disabling compacts and flushes for region at 1731118409776Disabling writes for close at 1731118409776Writing region close event to WAL at 1731118409777 (+1 ms)Running coprocessor post-close hooks at 1731118409782 (+5 ms)Closed at 1731118409782 2024-11-09T02:13:29,782 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1731118405247.ad814e81d5536ce802390dd5602e78a1. 2024-11-09T02:13:29,782 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/3a2a5d6e6a65b43660fcb665b4ef5c83/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,782 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5f3f9d6837573933dea12bd60465775e, disabling compactions & flushes 2024-11-09T02:13:29,782 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:29,782 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:29,782 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. after waiting 0 ms 2024-11-09T02:13:29,782 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:29,783 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:29,783 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3a2a5d6e6a65b43660fcb665b4ef5c83: Waiting for close lock at 1731118409778Running coprocessor pre-close hooks at 1731118409778Disabling compacts and flushes for region at 1731118409778Disabling writes for close at 1731118409778Writing region close event to WAL at 1731118409779 (+1 ms)Running coprocessor post-close hooks at 1731118409783 (+4 ms)Closed at 1731118409783 2024-11-09T02:13:29,783 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1731118405247.3a2a5d6e6a65b43660fcb665b4ef5c83. 2024-11-09T02:13:29,787 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/data/default/test/5f3f9d6837573933dea12bd60465775e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-09T02:13:29,788 INFO [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:29,788 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5f3f9d6837573933dea12bd60465775e: Waiting for close lock at 1731118409782Running coprocessor pre-close hooks at 1731118409782Disabling compacts and flushes for region at 1731118409782Disabling writes for close at 1731118409782Writing region close event to WAL at 1731118409783 (+1 ms)Running coprocessor post-close hooks at 1731118409788 (+5 ms)Closed at 1731118409788 2024-11-09T02:13:29,788 DEBUG [RS_CLOSE_REGION-regionserver/53f286d3af3b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1731118405247.5f3f9d6837573933dea12bd60465775e. 2024-11-09T02:13:29,886 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(976): stopping server 53f286d3af3b,46883,1731118402441; all regions closed. 2024-11-09T02:13:29,886 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(976): stopping server 53f286d3af3b,45941,1731118402318; all regions closed. 2024-11-09T02:13:29,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741865_1041 (size=4204) 2024-11-09T02:13:29,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741865_1041 (size=4204) 2024-11-09T02:13:29,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741834_1010 (size=7074) 2024-11-09T02:13:29,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741865_1041 (size=4204) 2024-11-09T02:13:29,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741834_1010 (size=7074) 2024-11-09T02:13:29,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741834_1010 (size=7074) 2024-11-09T02:13:29,892 DEBUG [RS:0;53f286d3af3b:45941 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/oldWALs 2024-11-09T02:13:29,892 INFO [RS:0;53f286d3af3b:45941 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 53f286d3af3b%2C45941%2C1731118402318.meta:.meta(num 1731118407387) 2024-11-09T02:13:29,893 DEBUG [RS:1;53f286d3af3b:46883 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/oldWALs 2024-11-09T02:13:29,893 INFO [RS:1;53f286d3af3b:46883 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 53f286d3af3b%2C46883%2C1731118402441:(num 1731118404218) 2024-11-09T02:13:29,893 DEBUG [RS:1;53f286d3af3b:46883 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:13:29,893 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T02:13:29,893 INFO [RS:1;53f286d3af3b:46883 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T02:13:29,893 INFO [RS:1;53f286d3af3b:46883 {}] hbase.ChoreService(370): Chore service for: regionserver/53f286d3af3b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T02:13:29,893 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-09T02:13:29,893 INFO [regionserver/53f286d3af3b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T02:13:29,893 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-09T02:13:29,893 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-09T02:13:29,893 INFO [RS:1;53f286d3af3b:46883 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T02:13:29,894 INFO [RS:1;53f286d3af3b:46883 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46883 2024-11-09T02:13:29,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741833_1009 (size=7064) 2024-11-09T02:13:29,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741833_1009 (size=7064) 2024-11-09T02:13:29,896 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/WALs/53f286d3af3b,45941,1731118402318/53f286d3af3b%2C45941%2C1731118402318.1731118404218 not finished, retry = 0 2024-11-09T02:13:29,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741833_1009 (size=7064) 2024-11-09T02:13:29,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T02:13:29,986 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/53f286d3af3b,46883,1731118402441 2024-11-09T02:13:29,987 INFO [RS:1;53f286d3af3b:46883 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T02:13:29,989 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [53f286d3af3b,46883,1731118402441] 2024-11-09T02:13:30,002 DEBUG [RS:0;53f286d3af3b:45941 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/oldWALs 2024-11-09T02:13:30,002 INFO [RS:0;53f286d3af3b:45941 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 53f286d3af3b%2C45941%2C1731118402318:(num 1731118404218) 2024-11-09T02:13:30,002 DEBUG [RS:0;53f286d3af3b:45941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-09T02:13:30,003 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.LeaseManager(133): Closed leases 2024-11-09T02:13:30,003 INFO [RS:0;53f286d3af3b:45941 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T02:13:30,003 INFO [RS:0;53f286d3af3b:45941 {}] hbase.ChoreService(370): Chore service for: regionserver/53f286d3af3b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-09T02:13:30,004 INFO [RS:0;53f286d3af3b:45941 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T02:13:30,004 INFO [regionserver/53f286d3af3b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T02:13:30,004 INFO [RS:0;53f286d3af3b:45941 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45941 2024-11-09T02:13:30,113 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/53f286d3af3b,46883,1731118402441 already deleted, retry=false 2024-11-09T02:13:30,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/53f286d3af3b,45941,1731118402318 2024-11-09T02:13:30,113 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-09T02:13:30,113 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 53f286d3af3b,46883,1731118402441 expired; onlineServers=1 2024-11-09T02:13:30,114 INFO [RS:0;53f286d3af3b:45941 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T02:13:30,115 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [53f286d3af3b,45941,1731118402318] 2024-11-09T02:13:30,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:13:30,178 INFO [RS:1;53f286d3af3b:46883 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T02:13:30,178 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46883-0x1011d5e7e540002, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:13:30,178 INFO [RS:1;53f286d3af3b:46883 {}] regionserver.HRegionServer(1031): Exiting; stopping=53f286d3af3b,46883,1731118402441; zookeeper connection closed. 2024-11-09T02:13:30,179 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@41bfccdc {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@41bfccdc 2024-11-09T02:13:30,310 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-09T02:13:30,333 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/53f286d3af3b,45941,1731118402318 already deleted, retry=false 2024-11-09T02:13:30,333 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 53f286d3af3b,45941,1731118402318 expired; onlineServers=0 2024-11-09T02:13:30,333 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '53f286d3af3b,36011,1731118401306' ***** 2024-11-09T02:13:30,333 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-09T02:13:30,334 INFO [M:0;53f286d3af3b:36011 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-09T02:13:30,334 INFO [M:0;53f286d3af3b:36011 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-09T02:13:30,334 DEBUG [M:0;53f286d3af3b:36011 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-09T02:13:30,334 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-09T02:13:30,334 DEBUG [M:0;53f286d3af3b:36011 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-09T02:13:30,334 DEBUG [master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.small.0-1731118403679 {}] cleaner.HFileCleaner(306): Exit Thread[master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.small.0-1731118403679,5,FailOnTimeoutGroup] 2024-11-09T02:13:30,334 DEBUG [master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.large.0-1731118403678 {}] cleaner.HFileCleaner(306): Exit Thread[master/53f286d3af3b:0:becomeActiveMaster-HFileCleaner.large.0-1731118403678,5,FailOnTimeoutGroup] 2024-11-09T02:13:30,334 INFO [M:0;53f286d3af3b:36011 {}] hbase.ChoreService(370): Chore service for: master/53f286d3af3b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-09T02:13:30,334 INFO [M:0;53f286d3af3b:36011 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-09T02:13:30,334 DEBUG [M:0;53f286d3af3b:36011 {}] master.HMaster(1795): Stopping service threads 2024-11-09T02:13:30,335 INFO [M:0;53f286d3af3b:36011 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-09T02:13:30,335 INFO [M:0;53f286d3af3b:36011 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-09T02:13:30,335 INFO [M:0;53f286d3af3b:36011 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-09T02:13:30,335 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-09T02:13:30,361 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-09T02:13:30,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:13:30,365 INFO [RS:0;53f286d3af3b:45941 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T02:13:30,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45941-0x1011d5e7e540001, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:13:30,365 INFO [RS:0;53f286d3af3b:45941 {}] regionserver.HRegionServer(1031): Exiting; stopping=53f286d3af3b,45941,1731118402318; zookeeper connection closed. 2024-11-09T02:13:30,366 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@32bd2304 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@32bd2304 2024-11-09T02:13:30,366 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-09T02:13:30,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-09T02:13:30,450 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-09T02:13:30,450 DEBUG [M:0;53f286d3af3b:36011 {}] zookeeper.ZKUtil(347): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-09T02:13:30,450 WARN [M:0;53f286d3af3b:36011 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-09T02:13:30,451 INFO [M:0;53f286d3af3b:36011 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/.lastflushedseqids 2024-11-09T02:13:30,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741870_1046 (size=1) 2024-11-09T02:13:30,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741870_1046 (size=1) 2024-11-09T02:13:30,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741870_1046 (size=1) 2024-11-09T02:13:30,462 INFO [M:0;53f286d3af3b:36011 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-09T02:13:30,462 INFO [M:0;53f286d3af3b:36011 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-09T02:13:30,462 DEBUG [M:0;53f286d3af3b:36011 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-09T02:13:30,462 INFO [M:0;53f286d3af3b:36011 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:13:30,462 DEBUG [M:0;53f286d3af3b:36011 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:13:30,462 DEBUG [M:0;53f286d3af3b:36011 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-09T02:13:30,462 DEBUG [M:0;53f286d3af3b:36011 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:13:30,462 INFO [M:0;53f286d3af3b:36011 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=250.11 KB heapSize=303.94 KB 2024-11-09T02:13:30,479 DEBUG [M:0;53f286d3af3b:36011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/892ff29a0eac4a77b8b1b22d58df5baf is 82, key is hbase:meta,,1/info:regioninfo/1731118407700/Put/seqid=0 2024-11-09T02:13:30,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741871_1047 (size=6071) 2024-11-09T02:13:30,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741871_1047 (size=6071) 2024-11-09T02:13:30,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741871_1047 (size=6071) 2024-11-09T02:13:30,488 INFO [M:0;53f286d3af3b:36011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/892ff29a0eac4a77b8b1b22d58df5baf 2024-11-09T02:13:30,514 DEBUG [M:0;53f286d3af3b:36011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/33ab1b78039e40eda6e6234b28004de5 is 2990, key is \x00\x00\x00\x00\x00\x00\x009/proc:d/1731118409653/Put/seqid=0 2024-11-09T02:13:30,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741872_1048 (size=12801) 2024-11-09T02:13:30,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741872_1048 (size=12801) 2024-11-09T02:13:30,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741872_1048 (size=12801) 2024-11-09T02:13:30,525 INFO [M:0;53f286d3af3b:36011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=248.83 KB at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/33ab1b78039e40eda6e6234b28004de5 2024-11-09T02:13:30,556 DEBUG [M:0;53f286d3af3b:36011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c6053ed0eb624dbb95f8b3d67af6e5b9 is 69, key is 53f286d3af3b,45941,1731118402318/rs:state/1731118403799/Put/seqid=0 2024-11-09T02:13:30,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741873_1049 (size=5442) 2024-11-09T02:13:30,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741873_1049 (size=5442) 2024-11-09T02:13:30,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741873_1049 (size=5442) 2024-11-09T02:13:30,565 INFO [M:0;53f286d3af3b:36011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c6053ed0eb624dbb95f8b3d67af6e5b9 2024-11-09T02:13:30,572 INFO [M:0;53f286d3af3b:36011 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c6053ed0eb624dbb95f8b3d67af6e5b9 2024-11-09T02:13:30,587 DEBUG [M:0;53f286d3af3b:36011 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/197b469f40bf4336ac17b9fbf90a5bcf is 52, key is load_balancer_on/state:d/1731118406574/Put/seqid=0 2024-11-09T02:13:30,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741874_1050 (size=5058) 2024-11-09T02:13:30,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39093 is added to blk_1073741874_1050 (size=5058) 2024-11-09T02:13:30,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741874_1050 (size=5058) 2024-11-09T02:13:30,597 INFO [M:0;53f286d3af3b:36011 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=48 B at sequenceid=575 (bloomFilter=true), to=hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/197b469f40bf4336ac17b9fbf90a5bcf 2024-11-09T02:13:30,604 DEBUG [M:0;53f286d3af3b:36011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/892ff29a0eac4a77b8b1b22d58df5baf as hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/892ff29a0eac4a77b8b1b22d58df5baf 2024-11-09T02:13:30,610 INFO [M:0;53f286d3af3b:36011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/892ff29a0eac4a77b8b1b22d58df5baf, entries=14, sequenceid=575, filesize=5.9 K 2024-11-09T02:13:30,612 DEBUG [M:0;53f286d3af3b:36011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/33ab1b78039e40eda6e6234b28004de5 as hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/33ab1b78039e40eda6e6234b28004de5 2024-11-09T02:13:30,619 INFO [M:0;53f286d3af3b:36011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/33ab1b78039e40eda6e6234b28004de5, entries=79, sequenceid=575, filesize=12.5 K 2024-11-09T02:13:30,621 DEBUG [M:0;53f286d3af3b:36011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/c6053ed0eb624dbb95f8b3d67af6e5b9 as hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c6053ed0eb624dbb95f8b3d67af6e5b9 2024-11-09T02:13:30,629 INFO [M:0;53f286d3af3b:36011 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for c6053ed0eb624dbb95f8b3d67af6e5b9 2024-11-09T02:13:30,629 INFO [M:0;53f286d3af3b:36011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/c6053ed0eb624dbb95f8b3d67af6e5b9, entries=3, sequenceid=575, filesize=5.3 K 2024-11-09T02:13:30,631 DEBUG [M:0;53f286d3af3b:36011 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/197b469f40bf4336ac17b9fbf90a5bcf as hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/197b469f40bf4336ac17b9fbf90a5bcf 2024-11-09T02:13:30,638 INFO [M:0;53f286d3af3b:36011 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45553/user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/197b469f40bf4336ac17b9fbf90a5bcf, entries=1, sequenceid=575, filesize=4.9 K 2024-11-09T02:13:30,639 INFO [M:0;53f286d3af3b:36011 {}] regionserver.HRegion(3140): Finished flush of dataSize ~250.11 KB/256108, heapSize ~303.88 KB/311168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 177ms, sequenceid=575, compaction requested=false 2024-11-09T02:13:30,641 INFO [M:0;53f286d3af3b:36011 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-09T02:13:30,641 DEBUG [M:0;53f286d3af3b:36011 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731118410462Disabling compacts and flushes for region at 1731118410462Disabling writes for close at 1731118410462Obtaining lock to block concurrent updates at 1731118410462Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731118410462Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=256108, getHeapSize=311168, getOffHeapSize=0, getCellsCount=714 at 1731118410463 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731118410463Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731118410463Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731118410479 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731118410479Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731118410497 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731118410514 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731118410514Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731118410534 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731118410555 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731118410555Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1731118410573 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1731118410587 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1731118410587Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3241b836: reopening flushed file at 1731118410603 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3affc8c3: reopening flushed file at 1731118410610 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b031c68: reopening flushed file at 1731118410619 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@514c186e: reopening flushed file at 1731118410630 (+11 ms)Finished flush of dataSize ~250.11 KB/256108, heapSize ~303.88 KB/311168, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 177ms, sequenceid=575, compaction requested=false at 1731118410639 (+9 ms)Writing region close event to WAL at 1731118410641 (+2 ms)Closed at 1731118410641 2024-11-09T02:13:30,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45399 is added to blk_1073741830_1006 (size=294492) 2024-11-09T02:13:30,644 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/c0186ae7-bbf4-379a-a01e-95d29ac0dfc8/MasterData/WALs/53f286d3af3b,36011,1731118401306/53f286d3af3b%2C36011%2C1731118401306.1731118402944 not finished, retry = 0 2024-11-09T02:13:30,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43193 is added to blk_1073741830_1006 (size=294492) 2024-11-09T02:13:30,746 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-09T02:13:30,746 INFO [M:0;53f286d3af3b:36011 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-09T02:13:30,746 INFO [M:0;53f286d3af3b:36011 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:36011 2024-11-09T02:13:30,746 INFO [M:0;53f286d3af3b:36011 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-09T02:13:30,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:13:30,920 INFO [M:0;53f286d3af3b:36011 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-09T02:13:30,920 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36011-0x1011d5e7e540000, quorum=127.0.0.1:59275, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-09T02:13:30,929 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@67ddfd25{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T02:13:30,933 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@30e38263{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T02:13:30,933 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T02:13:30,933 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@52cf89e6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T02:13:30,933 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@11ebaa12{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/hadoop.log.dir/,STOPPED} 2024-11-09T02:13:30,936 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T02:13:30,936 WARN [BP-306821699-172.17.0.3-1731118396170 heartbeating to localhost/127.0.0.1:45553 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T02:13:30,937 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T02:13:30,937 WARN [BP-306821699-172.17.0.3-1731118396170 heartbeating to localhost/127.0.0.1:45553 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-306821699-172.17.0.3-1731118396170 (Datanode Uuid c5bbc037-aae6-45bb-9ee0-c0a8ee5d9ef6) service to localhost/127.0.0.1:45553 2024-11-09T02:13:30,938 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data5/current/BP-306821699-172.17.0.3-1731118396170 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T02:13:30,939 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data6/current/BP-306821699-172.17.0.3-1731118396170 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T02:13:30,946 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T02:13:30,952 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7cfccf27{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T02:13:30,953 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@6c41bbb7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T02:13:30,953 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T02:13:30,953 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@12f16529{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T02:13:30,954 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@34a3e1ff{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/hadoop.log.dir/,STOPPED} 2024-11-09T02:13:30,959 WARN [BP-306821699-172.17.0.3-1731118396170 heartbeating to localhost/127.0.0.1:45553 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T02:13:30,959 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T02:13:30,959 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T02:13:30,959 WARN [BP-306821699-172.17.0.3-1731118396170 heartbeating to localhost/127.0.0.1:45553 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-306821699-172.17.0.3-1731118396170 (Datanode Uuid dd851afc-732d-4c4c-b263-dd62f61cbed1) service to localhost/127.0.0.1:45553 2024-11-09T02:13:30,960 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data3/current/BP-306821699-172.17.0.3-1731118396170 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T02:13:30,960 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data4/current/BP-306821699-172.17.0.3-1731118396170 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T02:13:30,960 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T02:13:30,969 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2226d6a6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-09T02:13:30,970 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1da1f773{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T02:13:30,970 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T02:13:30,970 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1c5d2af1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T02:13:30,970 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@739a104e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/hadoop.log.dir/,STOPPED} 2024-11-09T02:13:30,973 WARN [BP-306821699-172.17.0.3-1731118396170 heartbeating to localhost/127.0.0.1:45553 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-09T02:13:30,973 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-09T02:13:30,973 WARN [BP-306821699-172.17.0.3-1731118396170 heartbeating to localhost/127.0.0.1:45553 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-306821699-172.17.0.3-1731118396170 (Datanode Uuid cfe1fdc4-2d17-4683-9ab8-0963e6c7047c) service to localhost/127.0.0.1:45553 2024-11-09T02:13:30,973 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-09T02:13:30,973 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data1/current/BP-306821699-172.17.0.3-1731118396170 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T02:13:30,974 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-09T02:13:30,974 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/cluster_57b46723-60f1-f03d-3cfc-48075c5eb9b4/data/data2/current/BP-306821699-172.17.0.3-1731118396170 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-09T02:13:30,987 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@376fa8f3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-09T02:13:30,989 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@672b88ea{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-09T02:13:30,989 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-09T02:13:30,989 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@76beb053{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-09T02:13:30,989 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@265b24f1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/625eacd2-e71f-9b39-79f0-d667a84e07db/hadoop.log.dir/,STOPPED} 2024-11-09T02:13:31,008 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-09T02:13:31,070 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down