2024-11-20 12:13:25,907 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-20 12:13:25,922 main DEBUG Took 0.012746 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-20 12:13:25,923 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-20 12:13:25,923 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-20 12:13:25,924 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-20 12:13:25,925 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,934 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-20 12:13:25,950 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,952 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,953 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,954 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,957 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,957 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,959 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,959 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,960 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,961 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,962 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,963 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,963 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,964 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,965 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,965 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,966 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,966 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,967 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,967 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,968 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,969 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,969 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,970 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-20 12:13:25,970 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,971 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-20 12:13:25,973 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-20 12:13:25,975 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-20 12:13:25,978 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-20 12:13:25,979 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-20 12:13:25,980 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-20 12:13:25,980 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-20 12:13:25,991 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-20 12:13:25,993 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-20 12:13:25,995 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-20 12:13:25,996 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-20 12:13:25,997 main DEBUG createAppenders(={Console}) 2024-11-20 12:13:25,998 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 initialized 2024-11-20 12:13:25,998 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 2024-11-20 12:13:25,998 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@26abb146 OK. 2024-11-20 12:13:25,999 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-20 12:13:25,999 main DEBUG OutputStream closed 2024-11-20 12:13:25,999 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-20 12:13:26,000 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-20 12:13:26,000 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@c88a337 OK 2024-11-20 12:13:26,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-20 12:13:26,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-20 12:13:26,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-20 12:13:26,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-20 12:13:26,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-20 12:13:26,099 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-20 12:13:26,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-20 12:13:26,100 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-20 12:13:26,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-20 12:13:26,101 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-20 12:13:26,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-20 12:13:26,102 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-20 12:13:26,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-20 12:13:26,103 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-20 12:13:26,104 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-20 12:13:26,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-20 12:13:26,105 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-20 12:13:26,106 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-20 12:13:26,109 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20 12:13:26,109 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@13330ac6) with optional ClassLoader: null 2024-11-20 12:13:26,110 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-20 12:13:26,111 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@13330ac6] started OK. 2024-11-20T12:13:26,126 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP timeout: 13 mins 2024-11-20 12:13:26,129 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-20 12:13:26,129 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-20T12:13:26,372 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951 2024-11-20T12:13:26,393 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=class org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$HMasterForTest, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-20T12:13:26,416 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/cluster_eac485dc-d47e-5b3c-9347-7b92053ce28e, deleteOnExit=true 2024-11-20T12:13:26,416 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-20T12:13:26,418 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/test.cache.data in system properties and HBase conf 2024-11-20T12:13:26,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/hadoop.tmp.dir in system properties and HBase conf 2024-11-20T12:13:26,419 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/hadoop.log.dir in system properties and HBase conf 2024-11-20T12:13:26,420 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-20T12:13:26,421 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-20T12:13:26,422 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-20T12:13:26,520 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-20T12:13:26,616 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-20T12:13:26,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-20T12:13:26,622 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-20T12:13:26,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-20T12:13:26,623 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T12:13:26,624 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-20T12:13:26,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-20T12:13:26,625 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-20T12:13:26,626 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T12:13:26,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-20T12:13:26,627 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/nfs.dump.dir in system properties and HBase conf 2024-11-20T12:13:26,628 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/java.io.tmpdir in system properties and HBase conf 2024-11-20T12:13:26,629 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-20T12:13:26,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-20T12:13:26,630 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-20T12:13:27,804 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-20T12:13:27,872 INFO [Time-limited test {}] log.Log(170): Logging initialized @2928ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-20T12:13:27,941 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T12:13:27,997 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T12:13:28,017 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T12:13:28,017 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T12:13:28,018 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T12:13:28,029 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T12:13:28,032 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3912242{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/hadoop.log.dir/,AVAILABLE} 2024-11-20T12:13:28,033 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@595320c8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T12:13:28,203 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64418798{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/java.io.tmpdir/jetty-localhost-44829-hadoop-hdfs-3_4_1-tests_jar-_-any-14587008860393330386/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-20T12:13:28,212 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11744524{HTTP/1.1, (http/1.1)}{localhost:44829} 2024-11-20T12:13:28,213 INFO [Time-limited test {}] server.Server(415): Started @3269ms 2024-11-20T12:13:28,757 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T12:13:28,765 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T12:13:28,766 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T12:13:28,766 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T12:13:28,766 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T12:13:28,768 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3a300759{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/hadoop.log.dir/,AVAILABLE} 2024-11-20T12:13:28,768 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@474d518f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T12:13:28,870 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@79b30fb0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/java.io.tmpdir/jetty-localhost-35539-hadoop-hdfs-3_4_1-tests_jar-_-any-17825596552112974364/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T12:13:28,870 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@548cd725{HTTP/1.1, (http/1.1)}{localhost:35539} 2024-11-20T12:13:28,871 INFO [Time-limited test {}] server.Server(415): Started @3927ms 2024-11-20T12:13:28,920 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T12:13:29,033 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T12:13:29,038 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T12:13:29,039 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T12:13:29,040 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T12:13:29,040 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T12:13:29,041 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@29652cf4{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/hadoop.log.dir/,AVAILABLE} 2024-11-20T12:13:29,041 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1b0a32ba{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T12:13:29,142 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@34fcb585{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/java.io.tmpdir/jetty-localhost-46087-hadoop-hdfs-3_4_1-tests_jar-_-any-7189161359680283745/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T12:13:29,143 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@389bb6f7{HTTP/1.1, (http/1.1)}{localhost:46087} 2024-11-20T12:13:29,143 INFO [Time-limited test {}] server.Server(415): Started @4199ms 2024-11-20T12:13:29,145 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T12:13:29,217 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-20T12:13:29,223 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-20T12:13:29,224 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-20T12:13:29,224 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-20T12:13:29,224 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-20T12:13:29,230 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@290e2eec{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/hadoop.log.dir/,AVAILABLE} 2024-11-20T12:13:29,232 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@417efacb{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-20T12:13:29,332 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3c1f701{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/java.io.tmpdir/jetty-localhost-43493-hadoop-hdfs-3_4_1-tests_jar-_-any-6367591957881610399/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-20T12:13:29,333 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@3d599bb4{HTTP/1.1, (http/1.1)}{localhost:43493} 2024-11-20T12:13:29,334 INFO [Time-limited test {}] server.Server(415): Started @4390ms 2024-11-20T12:13:29,336 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-20T12:13:30,739 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/cluster_eac485dc-d47e-5b3c-9347-7b92053ce28e/data/data2/current/BP-1955269602-172.17.0.3-1732104807313/current, will proceed with Du for space computation calculation, 2024-11-20T12:13:30,739 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/cluster_eac485dc-d47e-5b3c-9347-7b92053ce28e/data/data3/current/BP-1955269602-172.17.0.3-1732104807313/current, will proceed with Du for space computation calculation, 2024-11-20T12:13:30,739 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/cluster_eac485dc-d47e-5b3c-9347-7b92053ce28e/data/data4/current/BP-1955269602-172.17.0.3-1732104807313/current, will proceed with Du for space computation calculation, 2024-11-20T12:13:30,739 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/cluster_eac485dc-d47e-5b3c-9347-7b92053ce28e/data/data1/current/BP-1955269602-172.17.0.3-1732104807313/current, will proceed with Du for space computation calculation, 2024-11-20T12:13:30,778 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T12:13:30,778 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T12:13:30,834 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd2f16a20cdd583ad with lease ID 0xac50400e3782b9df: Processing first storage report for DS-3793a777-6860-453f-b38c-f277d5ad850e from datanode DatanodeRegistration(127.0.0.1:38431, datanodeUuid=31e796d5-f85d-40d7-9631-55474c6d4445, infoPort=43069, infoSecurePort=0, ipcPort=43453, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313) 2024-11-20T12:13:30,835 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd2f16a20cdd583ad with lease ID 0xac50400e3782b9df: from storage DS-3793a777-6860-453f-b38c-f277d5ad850e node DatanodeRegistration(127.0.0.1:38431, datanodeUuid=31e796d5-f85d-40d7-9631-55474c6d4445, infoPort=43069, infoSecurePort=0, ipcPort=43453, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-20T12:13:30,836 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf5e42976b3df50b0 with lease ID 0xac50400e3782b9e0: Processing first storage report for DS-9ec5aa7e-1c59-486c-ad94-94a00f2e2c4d from datanode DatanodeRegistration(127.0.0.1:41703, datanodeUuid=4e15a1a0-8253-41af-bb99-f1607f5e662b, infoPort=38745, infoSecurePort=0, ipcPort=43421, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313) 2024-11-20T12:13:30,836 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5e42976b3df50b0 with lease ID 0xac50400e3782b9e0: from storage DS-9ec5aa7e-1c59-486c-ad94-94a00f2e2c4d node DatanodeRegistration(127.0.0.1:41703, datanodeUuid=4e15a1a0-8253-41af-bb99-f1607f5e662b, infoPort=38745, infoSecurePort=0, ipcPort=43421, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T12:13:30,837 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd2f16a20cdd583ad with lease ID 0xac50400e3782b9df: Processing first storage report for DS-df011250-52a5-4a6f-9ac8-60ea3bd09ce5 from datanode DatanodeRegistration(127.0.0.1:38431, datanodeUuid=31e796d5-f85d-40d7-9631-55474c6d4445, infoPort=43069, infoSecurePort=0, ipcPort=43453, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313) 2024-11-20T12:13:30,837 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd2f16a20cdd583ad with lease ID 0xac50400e3782b9df: from storage DS-df011250-52a5-4a6f-9ac8-60ea3bd09ce5 node DatanodeRegistration(127.0.0.1:38431, datanodeUuid=31e796d5-f85d-40d7-9631-55474c6d4445, infoPort=43069, infoSecurePort=0, ipcPort=43453, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-20T12:13:30,837 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf5e42976b3df50b0 with lease ID 0xac50400e3782b9e0: Processing first storage report for DS-2b490901-e161-4106-bad9-022b39484f33 from datanode DatanodeRegistration(127.0.0.1:41703, datanodeUuid=4e15a1a0-8253-41af-bb99-f1607f5e662b, infoPort=38745, infoSecurePort=0, ipcPort=43421, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313) 2024-11-20T12:13:30,838 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf5e42976b3df50b0 with lease ID 0xac50400e3782b9e0: from storage DS-2b490901-e161-4106-bad9-022b39484f33 node DatanodeRegistration(127.0.0.1:41703, datanodeUuid=4e15a1a0-8253-41af-bb99-f1607f5e662b, infoPort=38745, infoSecurePort=0, ipcPort=43421, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T12:13:30,910 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/cluster_eac485dc-d47e-5b3c-9347-7b92053ce28e/data/data5/current/BP-1955269602-172.17.0.3-1732104807313/current, will proceed with Du for space computation calculation, 2024-11-20T12:13:30,925 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/cluster_eac485dc-d47e-5b3c-9347-7b92053ce28e/data/data6/current/BP-1955269602-172.17.0.3-1732104807313/current, will proceed with Du for space computation calculation, 2024-11-20T12:13:31,003 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-20T12:13:31,010 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x963da2af86c28200 with lease ID 0xac50400e3782b9e1: Processing first storage report for DS-924c6f71-12d9-407c-a800-6de4979baefb from datanode DatanodeRegistration(127.0.0.1:34249, datanodeUuid=881b8950-62ec-4c90-b287-352738d7ad5a, infoPort=41027, infoSecurePort=0, ipcPort=35175, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313) 2024-11-20T12:13:31,011 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x963da2af86c28200 with lease ID 0xac50400e3782b9e1: from storage DS-924c6f71-12d9-407c-a800-6de4979baefb node DatanodeRegistration(127.0.0.1:34249, datanodeUuid=881b8950-62ec-4c90-b287-352738d7ad5a, infoPort=41027, infoSecurePort=0, ipcPort=35175, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T12:13:31,011 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x963da2af86c28200 with lease ID 0xac50400e3782b9e1: Processing first storage report for DS-01e63272-79f4-47ce-9669-7ea57317b681 from datanode DatanodeRegistration(127.0.0.1:34249, datanodeUuid=881b8950-62ec-4c90-b287-352738d7ad5a, infoPort=41027, infoSecurePort=0, ipcPort=35175, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313) 2024-11-20T12:13:31,012 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x963da2af86c28200 with lease ID 0xac50400e3782b9e1: from storage DS-01e63272-79f4-47ce-9669-7ea57317b681 node DatanodeRegistration(127.0.0.1:34249, datanodeUuid=881b8950-62ec-4c90-b287-352738d7ad5a, infoPort=41027, infoSecurePort=0, ipcPort=35175, storageInfo=lv=-57;cid=testClusterID;nsid=857166345;c=1732104807313), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-20T12:13:31,060 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951 2024-11-20T12:13:31,181 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/cluster_eac485dc-d47e-5b3c-9347-7b92053ce28e/zookeeper_0, clientPort=56800, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/cluster_eac485dc-d47e-5b3c-9347-7b92053ce28e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d9cfe001-d522-80e6-cfcc-c32762384951/cluster_eac485dc-d47e-5b3c-9347-7b92053ce28e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-20T12:13:31,195 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=56800 2024-11-20T12:13:31,212 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:13:31,216 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:13:31,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741825_1001 (size=7) 2024-11-20T12:13:31,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741825_1001 (size=7) 2024-11-20T12:13:31,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741825_1001 (size=7) 2024-11-20T12:13:32,012 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad with version=8 2024-11-20T12:13:32,012 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/hbase-staging 2024-11-20T12:13:32,112 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-20T12:13:32,478 INFO [Time-limited test {}] client.ConnectionUtils(128): master/843d83d51ba0:0 server-side Connection retries=45 2024-11-20T12:13:32,487 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:32,488 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:32,493 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T12:13:32,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:32,493 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T12:13:32,632 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-20T12:13:32,691 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-20T12:13:32,700 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-20T12:13:32,704 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T12:13:32,728 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 2339 (auto-detected) 2024-11-20T12:13:32,729 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-20T12:13:32,748 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:39157 2024-11-20T12:13:32,771 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:39157 connecting to ZooKeeper ensemble=127.0.0.1:56800 2024-11-20T12:13:32,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:391570x0, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T12:13:32,877 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:39157-0x1015829ee370000 connected 2024-11-20T12:13:33,080 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:13:33,083 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:13:33,094 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T12:13:33,099 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad, hbase.cluster.distributed=false 2024-11-20T12:13:33,126 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T12:13:33,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39157 2024-11-20T12:13:33,137 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39157 2024-11-20T12:13:33,138 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39157 2024-11-20T12:13:33,139 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39157 2024-11-20T12:13:33,140 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39157 2024-11-20T12:13:33,234 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/843d83d51ba0:0 server-side Connection retries=45 2024-11-20T12:13:33,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:33,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:33,236 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T12:13:33,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:33,236 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T12:13:33,239 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T12:13:33,241 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T12:13:33,242 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44663 2024-11-20T12:13:33,244 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44663 connecting to ZooKeeper ensemble=127.0.0.1:56800 2024-11-20T12:13:33,245 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:13:33,250 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:13:33,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:446630x0, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T12:13:33,268 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:446630x0, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T12:13:33,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44663-0x1015829ee370001 connected 2024-11-20T12:13:33,273 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T12:13:33,281 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T12:13:33,284 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T12:13:33,290 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T12:13:33,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44663 2024-11-20T12:13:33,291 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44663 2024-11-20T12:13:33,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44663 2024-11-20T12:13:33,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44663 2024-11-20T12:13:33,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44663 2024-11-20T12:13:33,308 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/843d83d51ba0:0 server-side Connection retries=45 2024-11-20T12:13:33,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:33,308 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:33,308 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T12:13:33,309 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:33,309 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T12:13:33,309 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T12:13:33,309 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T12:13:33,310 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35591 2024-11-20T12:13:33,312 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35591 connecting to ZooKeeper ensemble=127.0.0.1:56800 2024-11-20T12:13:33,313 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:13:33,316 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:13:33,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:355910x0, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T12:13:33,331 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:355910x0, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T12:13:33,331 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35591-0x1015829ee370002 connected 2024-11-20T12:13:33,332 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T12:13:33,332 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T12:13:33,333 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T12:13:33,335 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T12:13:33,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35591 2024-11-20T12:13:33,336 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35591 2024-11-20T12:13:33,338 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35591 2024-11-20T12:13:33,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35591 2024-11-20T12:13:33,339 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35591 2024-11-20T12:13:33,355 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/843d83d51ba0:0 server-side Connection retries=45 2024-11-20T12:13:33,355 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:33,355 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:33,355 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-20T12:13:33,355 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-20T12:13:33,356 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-20T12:13:33,356 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-20T12:13:33,356 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-20T12:13:33,357 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:38659 2024-11-20T12:13:33,359 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38659 connecting to ZooKeeper ensemble=127.0.0.1:56800 2024-11-20T12:13:33,360 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:13:33,362 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:13:33,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:386590x0, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-20T12:13:33,373 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:386590x0, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T12:13:33,373 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38659-0x1015829ee370003 connected 2024-11-20T12:13:33,374 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-20T12:13:33,376 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-20T12:13:33,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-20T12:13:33,379 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-20T12:13:33,380 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38659 2024-11-20T12:13:33,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38659 2024-11-20T12:13:33,381 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38659 2024-11-20T12:13:33,382 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38659 2024-11-20T12:13:33,383 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38659 2024-11-20T12:13:33,396 DEBUG [M:0;843d83d51ba0:39157 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;843d83d51ba0:39157 2024-11-20T12:13:33,397 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/843d83d51ba0,39157,1732104812301 2024-11-20T12:13:33,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:13:33,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:13:33,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:13:33,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:13:33,415 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/843d83d51ba0,39157,1732104812301 2024-11-20T12:13:33,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T12:13:33,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T12:13:33,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:33,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-20T12:13:33,446 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:33,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:33,447 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:33,448 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-20T12:13:33,450 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/843d83d51ba0,39157,1732104812301 from backup master directory 2024-11-20T12:13:33,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:13:33,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:13:33,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:13:33,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/843d83d51ba0,39157,1732104812301 2024-11-20T12:13:33,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-20T12:13:33,466 WARN [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T12:13:33,466 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=843d83d51ba0,39157,1732104812301 2024-11-20T12:13:33,468 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-20T12:13:33,469 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-20T12:13:33,531 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/hbase.id] with ID: 65dd5be0-324e-4e74-8a0c-bcc97a2997df 2024-11-20T12:13:33,531 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/.tmp/hbase.id 2024-11-20T12:13:33,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741826_1002 (size=42) 2024-11-20T12:13:33,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741826_1002 (size=42) 2024-11-20T12:13:33,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741826_1002 (size=42) 2024-11-20T12:13:33,546 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/.tmp/hbase.id]:[hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/hbase.id] 2024-11-20T12:13:33,594 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-20T12:13:33,599 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-20T12:13:33,617 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-20T12:13:33,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:33,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:33,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:33,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:33,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741827_1003 (size=196) 2024-11-20T12:13:33,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741827_1003 (size=196) 2024-11-20T12:13:33,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741827_1003 (size=196) 2024-11-20T12:13:33,659 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:13:33,661 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-20T12:13:33,674 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:13:33,678 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T12:13:33,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741828_1004 (size=1189) 2024-11-20T12:13:33,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741828_1004 (size=1189) 2024-11-20T12:13:33,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741828_1004 (size=1189) 2024-11-20T12:13:33,726 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/data/master/store 2024-11-20T12:13:33,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741829_1005 (size=34) 2024-11-20T12:13:33,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741829_1005 (size=34) 2024-11-20T12:13:33,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741829_1005 (size=34) 2024-11-20T12:13:33,753 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-20T12:13:33,756 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:33,757 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T12:13:33,757 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:13:33,757 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:13:33,759 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T12:13:33,759 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:13:33,759 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:13:33,760 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732104813757Disabling compacts and flushes for region at 1732104813757Disabling writes for close at 1732104813759 (+2 ms)Writing region close event to WAL at 1732104813759Closed at 1732104813759 2024-11-20T12:13:33,762 WARN [master/843d83d51ba0:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/data/master/store/.initializing 2024-11-20T12:13:33,763 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/WALs/843d83d51ba0,39157,1732104812301 2024-11-20T12:13:33,770 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T12:13:33,784 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=843d83d51ba0%2C39157%2C1732104812301, suffix=, logDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/WALs/843d83d51ba0,39157,1732104812301, archiveDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/oldWALs, maxLogs=10 2024-11-20T12:13:33,806 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/WALs/843d83d51ba0,39157,1732104812301/843d83d51ba0%2C39157%2C1732104812301.1732104813788, exclude list is [], retry=0 2024-11-20T12:13:33,809 WARN [IPC Server handler 1 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:33,809 WARN [IPC Server handler 1 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:33,810 WARN [IPC Server handler 1 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:33,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41703,DS-9ec5aa7e-1c59-486c-ad94-94a00f2e2c4d,DISK] 2024-11-20T12:13:33,823 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34249,DS-924c6f71-12d9-407c-a800-6de4979baefb,DISK] 2024-11-20T12:13:33,827 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-20T12:13:33,861 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/WALs/843d83d51ba0,39157,1732104812301/843d83d51ba0%2C39157%2C1732104812301.1732104813788 2024-11-20T12:13:33,862 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41027:41027),(127.0.0.1/127.0.0.1:38745:38745)] 2024-11-20T12:13:33,863 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:13:33,863 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:33,866 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,867 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,902 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,928 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-20T12:13:33,932 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:33,935 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:33,935 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,939 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-20T12:13:33,940 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:33,941 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:33,942 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,945 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-20T12:13:33,945 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:33,947 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:33,948 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,951 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-20T12:13:33,951 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:33,952 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:33,953 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,956 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,958 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,963 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,964 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,968 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T12:13:33,972 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-20T12:13:33,977 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:33,978 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58787942, jitterRate=-0.12399140000343323}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T12:13:33,986 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732104813879Initializing all the Stores at 1732104813881 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104813882 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104813882Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104813883 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104813883Cleaning up temporary data from old regions at 1732104813964 (+81 ms)Region opened successfully at 1732104813986 (+22 ms) 2024-11-20T12:13:33,988 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-20T12:13:34,023 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@276714c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=843d83d51ba0/172.17.0.3:0 2024-11-20T12:13:34,054 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-20T12:13:34,064 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:34,064 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:34,067 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:34,069 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T12:13:34,074 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T12:13:34,074 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-20T12:13:34,098 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-20T12:13:34,106 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-20T12:13:34,162 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-20T12:13:34,164 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-20T12:13:34,166 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-20T12:13:34,172 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-20T12:13:34,175 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-20T12:13:34,178 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-20T12:13:34,190 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-20T12:13:34,192 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-20T12:13:34,203 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-20T12:13:34,219 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-20T12:13:34,224 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-20T12:13:34,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T12:13:34,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T12:13:34,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T12:13:34,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-20T12:13:34,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,236 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,240 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=843d83d51ba0,39157,1732104812301, sessionid=0x1015829ee370000, setting cluster-up flag (Was=false) 2024-11-20T12:13:34,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,267 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,298 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-20T12:13:34,301 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=843d83d51ba0,39157,1732104812301 2024-11-20T12:13:34,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:34,351 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-20T12:13:34,352 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=843d83d51ba0,39157,1732104812301 2024-11-20T12:13:34,358 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-20T12:13:34,387 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.HRegionServer(746): ClusterId : 65dd5be0-324e-4e74-8a0c-bcc97a2997df 2024-11-20T12:13:34,389 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(746): ClusterId : 65dd5be0-324e-4e74-8a0c-bcc97a2997df 2024-11-20T12:13:34,389 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(746): ClusterId : 65dd5be0-324e-4e74-8a0c-bcc97a2997df 2024-11-20T12:13:34,389 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T12:13:34,389 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T12:13:34,389 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-20T12:13:34,413 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T12:13:34,413 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T12:13:34,413 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-20T12:13:34,413 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T12:13:34,413 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T12:13:34,413 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-20T12:13:34,434 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:34,436 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T12:13:34,436 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T12:13:34,436 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-20T12:13:34,437 DEBUG [RS:1;843d83d51ba0:35591 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1947318d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=843d83d51ba0/172.17.0.3:0 2024-11-20T12:13:34,437 DEBUG [RS:2;843d83d51ba0:38659 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@386b5d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=843d83d51ba0/172.17.0.3:0 2024-11-20T12:13:34,438 DEBUG [RS:0;843d83d51ba0:44663 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c25b186, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=843d83d51ba0/172.17.0.3:0 2024-11-20T12:13:34,447 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-20T12:13:34,455 DEBUG [RS:1;843d83d51ba0:35591 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;843d83d51ba0:35591 2024-11-20T12:13:34,455 DEBUG [RS:0;843d83d51ba0:44663 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;843d83d51ba0:44663 2024-11-20T12:13:34,456 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-20T12:13:34,459 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T12:13:34,459 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T12:13:34,459 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T12:13:34,459 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T12:13:34,459 DEBUG [RS:1;843d83d51ba0:35591 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T12:13:34,459 DEBUG [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T12:13:34,462 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.HRegionServer(2659): reportForDuty to master=843d83d51ba0,39157,1732104812301 with port=35591, startcode=1732104813307 2024-11-20T12:13:34,462 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(2659): reportForDuty to master=843d83d51ba0,39157,1732104812301 with port=44663, startcode=1732104813203 2024-11-20T12:13:34,463 DEBUG [RS:2;843d83d51ba0:38659 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;843d83d51ba0:38659 2024-11-20T12:13:34,464 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-20T12:13:34,464 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-20T12:13:34,464 DEBUG [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-20T12:13:34,465 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(2659): reportForDuty to master=843d83d51ba0,39157,1732104812301 with port=38659, startcode=1732104813354 2024-11-20T12:13:34,463 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 843d83d51ba0,39157,1732104812301 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-20T12:13:34,471 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/843d83d51ba0:0, corePoolSize=5, maxPoolSize=5 2024-11-20T12:13:34,472 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/843d83d51ba0:0, corePoolSize=5, maxPoolSize=5 2024-11-20T12:13:34,472 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/843d83d51ba0:0, corePoolSize=5, maxPoolSize=5 2024-11-20T12:13:34,472 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/843d83d51ba0:0, corePoolSize=5, maxPoolSize=5 2024-11-20T12:13:34,472 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/843d83d51ba0:0, corePoolSize=10, maxPoolSize=10 2024-11-20T12:13:34,472 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,473 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/843d83d51ba0:0, corePoolSize=2, maxPoolSize=2 2024-11-20T12:13:34,473 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,476 DEBUG [RS:2;843d83d51ba0:38659 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T12:13:34,476 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104844476 2024-11-20T12:13:34,476 DEBUG [RS:0;843d83d51ba0:44663 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T12:13:34,478 DEBUG [RS:1;843d83d51ba0:35591 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-20T12:13:34,478 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-20T12:13:34,480 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-20T12:13:34,481 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:34,482 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-20T12:13:34,483 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-20T12:13:34,484 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-20T12:13:34,484 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-20T12:13:34,484 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-20T12:13:34,485 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,488 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:34,489 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-20T12:13:34,489 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-20T12:13:34,490 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-20T12:13:34,491 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-20T12:13:34,496 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-20T12:13:34,497 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-20T12:13:34,499 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.large.0-1732104814498,5,FailOnTimeoutGroup] 2024-11-20T12:13:34,500 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.small.0-1732104814499,5,FailOnTimeoutGroup] 2024-11-20T12:13:34,500 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,501 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-20T12:13:34,502 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,502 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,513 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46693, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T12:13:34,513 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39603, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T12:13:34,513 INFO [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49057, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-20T12:13:34,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741831_1007 (size=1321) 2024-11-20T12:13:34,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741831_1007 (size=1321) 2024-11-20T12:13:34,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741831_1007 (size=1321) 2024-11-20T12:13:34,521 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 843d83d51ba0,35591,1732104813307 2024-11-20T12:13:34,522 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-20T12:13:34,522 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:34,524 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] master.ServerManager(517): Registering regionserver=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:34,534 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 843d83d51ba0,38659,1732104813354 2024-11-20T12:13:34,535 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] master.ServerManager(517): Registering regionserver=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:34,538 DEBUG [RS:1;843d83d51ba0:35591 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:34,538 DEBUG [RS:1;843d83d51ba0:35591 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42641 2024-11-20T12:13:34,538 DEBUG [RS:1;843d83d51ba0:35591 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T12:13:34,540 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 843d83d51ba0,44663,1732104813203 2024-11-20T12:13:34,540 DEBUG [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:34,540 DEBUG [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42641 2024-11-20T12:13:34,540 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] master.ServerManager(517): Registering regionserver=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:34,540 DEBUG [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T12:13:34,543 DEBUG [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:34,544 DEBUG [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:42641 2024-11-20T12:13:34,544 DEBUG [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-20T12:13:34,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741832_1008 (size=32) 2024-11-20T12:13:34,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741832_1008 (size=32) 2024-11-20T12:13:34,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741832_1008 (size=32) 2024-11-20T12:13:34,550 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:34,551 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T12:13:34,553 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T12:13:34,555 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T12:13:34,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:34,556 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:34,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T12:13:34,559 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T12:13:34,560 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:34,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:34,561 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T12:13:34,564 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T12:13:34,564 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:34,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:34,565 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T12:13:34,568 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T12:13:34,568 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:34,569 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:34,569 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T12:13:34,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740 2024-11-20T12:13:34,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740 2024-11-20T12:13:34,574 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T12:13:34,574 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T12:13:34,575 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T12:13:34,577 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T12:13:34,581 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:34,582 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68891053, jitterRate=0.02655668556690216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T12:13:34,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732104814550Initializing all the Stores at 1732104814552 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104814552Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104814552Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104814552Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104814553 (+1 ms)Cleaning up temporary data from old regions at 1732104814574 (+21 ms)Region opened successfully at 1732104814588 (+14 ms) 2024-11-20T12:13:34,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T12:13:34,588 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T12:13:34,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T12:13:34,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T12:13:34,589 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T12:13:34,589 DEBUG [RS:1;843d83d51ba0:35591 {}] zookeeper.ZKUtil(111): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/843d83d51ba0,35591,1732104813307 2024-11-20T12:13:34,590 WARN [RS:1;843d83d51ba0:35591 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T12:13:34,590 INFO [RS:1;843d83d51ba0:35591 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T12:13:34,590 DEBUG [RS:0;843d83d51ba0:44663 {}] zookeeper.ZKUtil(111): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/843d83d51ba0,44663,1732104813203 2024-11-20T12:13:34,590 DEBUG [RS:1;843d83d51ba0:35591 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307 2024-11-20T12:13:34,590 WARN [RS:0;843d83d51ba0:44663 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T12:13:34,590 INFO [RS:0;843d83d51ba0:44663 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T12:13:34,591 DEBUG [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,44663,1732104813203 2024-11-20T12:13:34,592 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [843d83d51ba0,38659,1732104813354] 2024-11-20T12:13:34,592 DEBUG [RS:2;843d83d51ba0:38659 {}] zookeeper.ZKUtil(111): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/843d83d51ba0,38659,1732104813354 2024-11-20T12:13:34,592 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [843d83d51ba0,44663,1732104813203] 2024-11-20T12:13:34,592 WARN [RS:2;843d83d51ba0:38659 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-20T12:13:34,592 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [843d83d51ba0,35591,1732104813307] 2024-11-20T12:13:34,592 INFO [RS:2;843d83d51ba0:38659 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T12:13:34,592 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T12:13:34,592 DEBUG [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,38659,1732104813354 2024-11-20T12:13:34,592 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732104814588Disabling compacts and flushes for region at 1732104814588Disabling writes for close at 1732104814589 (+1 ms)Writing region close event to WAL at 1732104814591 (+2 ms)Closed at 1732104814592 (+1 ms) 2024-11-20T12:13:34,596 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:34,596 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-20T12:13:34,606 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T12:13:34,620 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:34,624 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-20T12:13:34,629 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T12:13:34,629 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T12:13:34,629 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-20T12:13:34,650 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T12:13:34,652 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T12:13:34,656 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-20T12:13:34,659 INFO [RS:0;843d83d51ba0:44663 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T12:13:34,659 INFO [RS:2;843d83d51ba0:38659 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T12:13:34,659 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,659 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,659 INFO [RS:1;843d83d51ba0:35591 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-20T12:13:34,659 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,664 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T12:13:34,664 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T12:13:34,664 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-20T12:13:34,672 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T12:13:34,672 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T12:13:34,673 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-20T12:13:34,674 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,674 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,674 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,674 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,675 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,675 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,675 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,675 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,675 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,675 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,675 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,675 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,675 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0, corePoolSize=2, maxPoolSize=2 2024-11-20T12:13:34,676 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0, corePoolSize=2, maxPoolSize=2 2024-11-20T12:13:34,676 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,676 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,677 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,677 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,677 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/843d83d51ba0:0, corePoolSize=3, maxPoolSize=3 2024-11-20T12:13:34,677 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,677 DEBUG [RS:0;843d83d51ba0:44663 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0, corePoolSize=3, maxPoolSize=3 2024-11-20T12:13:34,677 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/843d83d51ba0:0, corePoolSize=3, maxPoolSize=3 2024-11-20T12:13:34,677 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,677 DEBUG [RS:2;843d83d51ba0:38659 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0, corePoolSize=3, maxPoolSize=3 2024-11-20T12:13:34,677 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,677 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0, corePoolSize=2, maxPoolSize=2 2024-11-20T12:13:34,677 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,678 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,678 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,678 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,678 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,678 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/843d83d51ba0:0, corePoolSize=1, maxPoolSize=1 2024-11-20T12:13:34,678 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/843d83d51ba0:0, corePoolSize=3, maxPoolSize=3 2024-11-20T12:13:34,678 DEBUG [RS:1;843d83d51ba0:35591 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/843d83d51ba0:0, corePoolSize=3, maxPoolSize=3 2024-11-20T12:13:34,680 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,680 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,680 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,680 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,681 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,681 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,38659,1732104813354-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T12:13:34,684 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,684 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,684 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,685 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,685 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,685 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,44663,1732104813203-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T12:13:34,696 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,696 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,697 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,697 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,697 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,697 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,35591,1732104813307-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T12:13:34,714 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T12:13:34,715 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T12:13:34,717 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,44663,1732104813203-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,717 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,38659,1732104813354-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,718 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,718 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.Replication(171): 843d83d51ba0,38659,1732104813354 started 2024-11-20T12:13:34,722 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-20T12:13:34,722 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,722 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,35591,1732104813307-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,722 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.Replication(171): 843d83d51ba0,44663,1732104813203 started 2024-11-20T12:13:34,723 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,723 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.Replication(171): 843d83d51ba0,35591,1732104813307 started 2024-11-20T12:13:34,744 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,745 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(1482): Serving as 843d83d51ba0,38659,1732104813354, RpcServer on 843d83d51ba0/172.17.0.3:38659, sessionid=0x1015829ee370003 2024-11-20T12:13:34,745 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,745 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(1482): Serving as 843d83d51ba0,44663,1732104813203, RpcServer on 843d83d51ba0/172.17.0.3:44663, sessionid=0x1015829ee370001 2024-11-20T12:13:34,746 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T12:13:34,746 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T12:13:34,746 DEBUG [RS:2;843d83d51ba0:38659 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 843d83d51ba0,38659,1732104813354 2024-11-20T12:13:34,746 DEBUG [RS:0;843d83d51ba0:44663 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 843d83d51ba0,44663,1732104813203 2024-11-20T12:13:34,746 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '843d83d51ba0,38659,1732104813354' 2024-11-20T12:13:34,746 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '843d83d51ba0,44663,1732104813203' 2024-11-20T12:13:34,746 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T12:13:34,746 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T12:13:34,747 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T12:13:34,747 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T12:13:34,748 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T12:13:34,748 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T12:13:34,748 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T12:13:34,748 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T12:13:34,748 DEBUG [RS:2;843d83d51ba0:38659 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 843d83d51ba0,38659,1732104813354 2024-11-20T12:13:34,748 DEBUG [RS:0;843d83d51ba0:44663 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 843d83d51ba0,44663,1732104813203 2024-11-20T12:13:34,749 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '843d83d51ba0,44663,1732104813203' 2024-11-20T12:13:34,749 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '843d83d51ba0,38659,1732104813354' 2024-11-20T12:13:34,749 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T12:13:34,749 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T12:13:34,749 INFO [RS:1;843d83d51ba0:35591 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:34,749 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.HRegionServer(1482): Serving as 843d83d51ba0,35591,1732104813307, RpcServer on 843d83d51ba0/172.17.0.3:35591, sessionid=0x1015829ee370002 2024-11-20T12:13:34,749 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-20T12:13:34,749 DEBUG [RS:1;843d83d51ba0:35591 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 843d83d51ba0,35591,1732104813307 2024-11-20T12:13:34,749 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T12:13:34,749 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T12:13:34,750 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '843d83d51ba0,35591,1732104813307' 2024-11-20T12:13:34,750 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-20T12:13:34,750 DEBUG [RS:0;843d83d51ba0:44663 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T12:13:34,750 INFO [RS:0;843d83d51ba0:44663 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T12:13:34,750 INFO [RS:0;843d83d51ba0:44663 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T12:13:34,756 DEBUG [RS:2;843d83d51ba0:38659 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T12:13:34,756 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-20T12:13:34,757 INFO [RS:2;843d83d51ba0:38659 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T12:13:34,757 INFO [RS:2;843d83d51ba0:38659 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T12:13:34,757 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-20T12:13:34,757 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-20T12:13:34,757 DEBUG [RS:1;843d83d51ba0:35591 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 843d83d51ba0,35591,1732104813307 2024-11-20T12:13:34,757 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '843d83d51ba0,35591,1732104813307' 2024-11-20T12:13:34,758 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-20T12:13:34,758 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-20T12:13:34,759 DEBUG [RS:1;843d83d51ba0:35591 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-20T12:13:34,759 INFO [RS:1;843d83d51ba0:35591 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-20T12:13:34,759 INFO [RS:1;843d83d51ba0:35591 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-20T12:13:34,775 WARN [843d83d51ba0:39157 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-20T12:13:34,855 INFO [RS:0;843d83d51ba0:44663 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T12:13:34,857 INFO [RS:2;843d83d51ba0:38659 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T12:13:34,859 INFO [RS:0;843d83d51ba0:44663 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=843d83d51ba0%2C44663%2C1732104813203, suffix=, logDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,44663,1732104813203, archiveDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/oldWALs, maxLogs=32 2024-11-20T12:13:34,860 INFO [RS:1;843d83d51ba0:35591 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-20T12:13:34,861 INFO [RS:2;843d83d51ba0:38659 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=843d83d51ba0%2C38659%2C1732104813354, suffix=, logDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,38659,1732104813354, archiveDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/oldWALs, maxLogs=32 2024-11-20T12:13:34,863 INFO [RS:1;843d83d51ba0:35591 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=843d83d51ba0%2C35591%2C1732104813307, suffix=, logDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307, archiveDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/oldWALs, maxLogs=32 2024-11-20T12:13:34,881 DEBUG [RS:2;843d83d51ba0:38659 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,38659,1732104813354/843d83d51ba0%2C38659%2C1732104813354.1732104814863, exclude list is [], retry=0 2024-11-20T12:13:34,884 WARN [IPC Server handler 4 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:34,885 WARN [IPC Server handler 4 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:34,885 WARN [IPC Server handler 4 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:34,888 DEBUG [RS:1;843d83d51ba0:35591 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307/843d83d51ba0%2C35591%2C1732104813307.1732104814865, exclude list is [], retry=0 2024-11-20T12:13:34,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41703,DS-9ec5aa7e-1c59-486c-ad94-94a00f2e2c4d,DISK] 2024-11-20T12:13:34,888 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38431,DS-3793a777-6860-453f-b38c-f277d5ad850e,DISK] 2024-11-20T12:13:34,890 DEBUG [RS:0;843d83d51ba0:44663 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,44663,1732104813203/843d83d51ba0%2C44663%2C1732104813203.1732104814863, exclude list is [], retry=0 2024-11-20T12:13:34,893 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:34,893 WARN [IPC Server handler 3 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:34,893 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:34,894 INFO [RS:2;843d83d51ba0:38659 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,38659,1732104813354/843d83d51ba0%2C38659%2C1732104813354.1732104814863 2024-11-20T12:13:34,896 DEBUG [RS:2;843d83d51ba0:38659 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43069:43069),(127.0.0.1/127.0.0.1:38745:38745)] 2024-11-20T12:13:34,945 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34249,DS-924c6f71-12d9-407c-a800-6de4979baefb,DISK] 2024-11-20T12:13:34,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41703,DS-9ec5aa7e-1c59-486c-ad94-94a00f2e2c4d,DISK] 2024-11-20T12:13:34,946 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41703,DS-9ec5aa7e-1c59-486c-ad94-94a00f2e2c4d,DISK] 2024-11-20T12:13:34,947 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38431,DS-3793a777-6860-453f-b38c-f277d5ad850e,DISK] 2024-11-20T12:13:34,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38431,DS-3793a777-6860-453f-b38c-f277d5ad850e,DISK] 2024-11-20T12:13:34,952 INFO [RS:0;843d83d51ba0:44663 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,44663,1732104813203/843d83d51ba0%2C44663%2C1732104813203.1732104814863 2024-11-20T12:13:34,952 INFO [RS:1;843d83d51ba0:35591 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307/843d83d51ba0%2C35591%2C1732104813307.1732104814865 2024-11-20T12:13:34,952 DEBUG [RS:0;843d83d51ba0:44663 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41027:41027),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:43069:43069)] 2024-11-20T12:13:34,952 DEBUG [RS:1;843d83d51ba0:35591 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:43069:43069)] 2024-11-20T12:13:35,027 DEBUG [843d83d51ba0:39157 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-20T12:13:35,034 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(204): Hosts are {843d83d51ba0=0} racks are {/default-rack=0} 2024-11-20T12:13:35,040 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T12:13:35,040 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T12:13:35,040 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T12:13:35,040 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T12:13:35,040 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T12:13:35,040 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T12:13:35,040 INFO [843d83d51ba0:39157 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T12:13:35,040 INFO [843d83d51ba0:39157 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T12:13:35,040 INFO [843d83d51ba0:39157 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T12:13:35,040 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T12:13:35,047 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:35,054 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,35591,1732104813307, state=OPENING 2024-11-20T12:13:35,109 DEBUG [PEWorker-1 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-20T12:13:35,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:35,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:35,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:35,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:35,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:35,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:35,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:35,121 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:35,122 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:35,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,35591,1732104813307}] 2024-11-20T12:13:35,298 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T12:13:35,300 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33353, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T12:13:35,314 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T12:13:35,314 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T12:13:35,315 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T12:13:35,318 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=843d83d51ba0%2C35591%2C1732104813307.meta, suffix=.meta, logDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307, archiveDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/oldWALs, maxLogs=32 2024-11-20T12:13:35,333 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, exclude list is [], retry=0 2024-11-20T12:13:35,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34249,DS-924c6f71-12d9-407c-a800-6de4979baefb,DISK] 2024-11-20T12:13:35,337 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41703,DS-9ec5aa7e-1c59-486c-ad94-94a00f2e2c4d,DISK] 2024-11-20T12:13:35,338 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38431,DS-3793a777-6860-453f-b38c-f277d5ad850e,DISK] 2024-11-20T12:13:35,341 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta 2024-11-20T12:13:35,341 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41027:41027),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:43069:43069)] 2024-11-20T12:13:35,342 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:13:35,344 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T12:13:35,347 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T12:13:35,352 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T12:13:35,358 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T12:13:35,359 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,359 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T12:13:35,359 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T12:13:35,363 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T12:13:35,364 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T12:13:35,364 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:35,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:35,365 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T12:13:35,367 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T12:13:35,367 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:35,368 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:35,368 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T12:13:35,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T12:13:35,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:35,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:35,371 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T12:13:35,373 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T12:13:35,373 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:35,375 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:35,375 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T12:13:35,376 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740 2024-11-20T12:13:35,379 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740 2024-11-20T12:13:35,382 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T12:13:35,382 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T12:13:35,383 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T12:13:35,386 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T12:13:35,388 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68821506, jitterRate=0.025520354509353638}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T12:13:35,389 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T12:13:35,390 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732104815360Writing region info on filesystem at 1732104815360Initializing all the Stores at 1732104815362 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104815362Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104815363 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104815363Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104815363Cleaning up temporary data from old regions at 1732104815382 (+19 ms)Running coprocessor post-open hooks at 1732104815389 (+7 ms)Region opened successfully at 1732104815390 (+1 ms) 2024-11-20T12:13:35,398 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732104815291 2024-11-20T12:13:35,410 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T12:13:35,411 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T12:13:35,412 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:35,415 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,35591,1732104813307, state=OPEN 2024-11-20T12:13:35,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:35,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:35,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:35,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:35,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:35,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:35,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:35,425 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:35,426 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:35,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-20T12:13:35,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,35591,1732104813307 in 303 msec 2024-11-20T12:13:35,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-20T12:13:35,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 828 msec 2024-11-20T12:13:35,439 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:35,439 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-20T12:13:35,456 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:13:35,457 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1] 2024-11-20T12:13:35,477 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:13:35,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53501, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:13:35,500 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1130 sec 2024-11-20T12:13:35,501 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732104815501, completionTime=-1 2024-11-20T12:13:35,504 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-20T12:13:35,505 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:35,534 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-20T12:13:35,534 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104875534 2024-11-20T12:13:35,534 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104935534 2024-11-20T12:13:35,535 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 30 msec 2024-11-20T12:13:35,536 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:35,544 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,39157,1732104812301-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:35,544 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,39157,1732104812301-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:35,544 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,39157,1732104812301-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:35,546 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-843d83d51ba0:39157, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:35,546 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:35,547 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:35,552 DEBUG [master/843d83d51ba0:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-20T12:13:35,575 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.109sec 2024-11-20T12:13:35,577 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-20T12:13:35,579 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-20T12:13:35,580 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-20T12:13:35,580 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-20T12:13:35,580 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-20T12:13:35,581 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,39157,1732104812301-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-20T12:13:35,582 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,39157,1732104812301-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-20T12:13:35,585 DEBUG [master/843d83d51ba0:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-20T12:13:35,586 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-20T12:13:35,587 INFO [master/843d83d51ba0:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=843d83d51ba0,39157,1732104812301-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-20T12:13:35,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9330f7a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:13:35,600 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-20T12:13:35,600 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-20T12:13:35,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 843d83d51ba0,39157,-1 for getting cluster id 2024-11-20T12:13:35,605 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-20T12:13:35,613 DEBUG [HMasterForTest-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '65dd5be0-324e-4e74-8a0c-bcc97a2997df' 2024-11-20T12:13:35,615 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-20T12:13:35,615 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "65dd5be0-324e-4e74-8a0c-bcc97a2997df" 2024-11-20T12:13:35,615 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a761b17, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:13:35,615 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [843d83d51ba0,39157,-1] 2024-11-20T12:13:35,618 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-20T12:13:35,620 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:13:35,620 INFO [HMasterForTest-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48090, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-20T12:13:35,623 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6982d266, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-20T12:13:35,624 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:13:35,631 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1] 2024-11-20T12:13:35,632 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:13:35,634 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39898, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:13:35,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=843d83d51ba0,39157,1732104812301 2024-11-20T12:13:35,660 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-20T12:13:35,664 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 843d83d51ba0,39157,1732104812301 2024-11-20T12:13:35,666 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6a79697a 2024-11-20T12:13:35,667 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-20T12:13:35,670 INFO [HMasterForTest-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48106, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-20T12:13:35,677 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'test', {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-20T12:13:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:35,691 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_PRE_OPERATION 2024-11-20T12:13:35,694 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "test" procId is: 4 2024-11-20T12:13:35,694 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:35,698 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-20T12:13:35,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:13:35,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741837_1013 (size=358) 2024-11-20T12:13:35,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741837_1013 (size=358) 2024-11-20T12:13:35,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741837_1013 (size=358) 2024-11-20T12:13:35,722 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => ae055eeceea830ff461b6b4dc158461e, NAME => 'test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e.', STARTKEY => 'ccc', ENDKEY => 'ddd'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,723 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 99f9c663ffb10623454992ca39cafe6e, NAME => 'test,,1732104815671.99f9c663ffb10623454992ca39cafe6e.', STARTKEY => '', ENDKEY => 'bbb'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,723 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 23c5a5a22a2d09813d1ff472bb76f407, NAME => 'test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407.', STARTKEY => 'bbb', ENDKEY => 'ccc'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741838_1014 (size=45) 2024-11-20T12:13:35,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741838_1014 (size=45) 2024-11-20T12:13:35,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741838_1014 (size=45) 2024-11-20T12:13:35,747 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,748 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing ae055eeceea830ff461b6b4dc158461e, disabling compactions & flushes 2024-11-20T12:13:35,748 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:35,748 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:35,748 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. after waiting 0 ms 2024-11-20T12:13:35,748 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:35,748 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:35,748 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for ae055eeceea830ff461b6b4dc158461e: Waiting for close lock at 1732104815748Disabling compacts and flushes for region at 1732104815748Disabling writes for close at 1732104815748Writing region close event to WAL at 1732104815748Closed at 1732104815748 2024-11-20T12:13:35,749 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 9b2ba9c7430b8a524b3966df315ad45c, NAME => 'test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c.', STARTKEY => 'ddd', ENDKEY => 'eee'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741839_1015 (size=42) 2024-11-20T12:13:35,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741839_1015 (size=42) 2024-11-20T12:13:35,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741839_1015 (size=42) 2024-11-20T12:13:35,755 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,,1732104815671.99f9c663ffb10623454992ca39cafe6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,755 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 99f9c663ffb10623454992ca39cafe6e, disabling compactions & flushes 2024-11-20T12:13:35,755 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:35,755 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:35,755 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. after waiting 0 ms 2024-11-20T12:13:35,755 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:35,755 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:35,756 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 99f9c663ffb10623454992ca39cafe6e: Waiting for close lock at 1732104815755Disabling compacts and flushes for region at 1732104815755Disabling writes for close at 1732104815755Writing region close event to WAL at 1732104815755Closed at 1732104815755 2024-11-20T12:13:35,757 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8448ac25ea038ac8c63c9da8906d8673, NAME => 'test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673.', STARTKEY => 'eee', ENDKEY => 'fff'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,758 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,758 WARN [IPC Server handler 3 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,759 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741840_1016 (size=45) 2024-11-20T12:13:35,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741840_1016 (size=45) 2024-11-20T12:13:35,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741840_1016 (size=45) 2024-11-20T12:13:35,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 23c5a5a22a2d09813d1ff472bb76f407, disabling compactions & flushes 2024-11-20T12:13:35,764 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:35,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:35,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. after waiting 0 ms 2024-11-20T12:13:35,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:35,764 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:35,764 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 23c5a5a22a2d09813d1ff472bb76f407: Waiting for close lock at 1732104815764Disabling compacts and flushes for region at 1732104815764Disabling writes for close at 1732104815764Writing region close event to WAL at 1732104815764Closed at 1732104815764 2024-11-20T12:13:35,765 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 325e5f934bae06dfb2390407a9e9726f, NAME => 'test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f.', STARTKEY => 'fff', ENDKEY => 'ggg'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,769 WARN [IPC Server handler 4 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,769 WARN [IPC Server handler 4 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,770 WARN [IPC Server handler 4 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,775 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,775 WARN [IPC Server handler 0 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,775 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741841_1017 (size=45) 2024-11-20T12:13:35,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741841_1017 (size=45) 2024-11-20T12:13:35,781 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,781 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 9b2ba9c7430b8a524b3966df315ad45c, disabling compactions & flushes 2024-11-20T12:13:35,781 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:35,781 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:35,781 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. after waiting 0 ms 2024-11-20T12:13:35,781 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:35,781 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:35,781 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 9b2ba9c7430b8a524b3966df315ad45c: Waiting for close lock at 1732104815781Disabling compacts and flushes for region at 1732104815781Disabling writes for close at 1732104815781Writing region close event to WAL at 1732104815781Closed at 1732104815781 2024-11-20T12:13:35,782 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => abac5b1bb17e965219c5ca01a8ebc325, NAME => 'test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325.', STARTKEY => 'ggg', ENDKEY => 'hhh'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741842_1018 (size=45) 2024-11-20T12:13:35,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741842_1018 (size=45) 2024-11-20T12:13:35,792 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,792 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8448ac25ea038ac8c63c9da8906d8673, disabling compactions & flushes 2024-11-20T12:13:35,792 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:35,792 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:35,792 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. after waiting 0 ms 2024-11-20T12:13:35,792 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:35,793 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:35,793 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8448ac25ea038ac8c63c9da8906d8673: Waiting for close lock at 1732104815792Disabling compacts and flushes for region at 1732104815792Disabling writes for close at 1732104815792Writing region close event to WAL at 1732104815792Closed at 1732104815792 2024-11-20T12:13:35,794 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6171c0ffde5383e6eb4635a4c76e3ad1, NAME => 'test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1.', STARTKEY => 'hhh', ENDKEY => 'iii'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,804 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,805 WARN [IPC Server handler 0 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,805 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,805 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,805 WARN [IPC Server handler 3 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,805 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741843_1019 (size=45) 2024-11-20T12:13:35,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741843_1019 (size=45) 2024-11-20T12:13:35,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:13:35,810 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,810 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 325e5f934bae06dfb2390407a9e9726f, disabling compactions & flushes 2024-11-20T12:13:35,810 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:35,810 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:35,810 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. after waiting 0 ms 2024-11-20T12:13:35,810 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:35,810 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:35,811 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 325e5f934bae06dfb2390407a9e9726f: Waiting for close lock at 1732104815810Disabling compacts and flushes for region at 1732104815810Disabling writes for close at 1732104815810Writing region close event to WAL at 1732104815810Closed at 1732104815810 2024-11-20T12:13:35,812 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => dec6f7c7f05d870bae6d81029b3676e2, NAME => 'test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2.', STARTKEY => 'iii', ENDKEY => 'jjj'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,821 WARN [IPC Server handler 1 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,822 WARN [IPC Server handler 1 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,822 WARN [IPC Server handler 1 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741845_1021 (size=45) 2024-11-20T12:13:35,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741845_1021 (size=45) 2024-11-20T12:13:35,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741844_1020 (size=45) 2024-11-20T12:13:35,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741844_1020 (size=45) 2024-11-20T12:13:35,832 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,833 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 6171c0ffde5383e6eb4635a4c76e3ad1, disabling compactions & flushes 2024-11-20T12:13:35,833 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:35,833 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:35,833 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. after waiting 0 ms 2024-11-20T12:13:35,833 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:35,833 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:35,833 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6171c0ffde5383e6eb4635a4c76e3ad1: Waiting for close lock at 1732104815832Disabling compacts and flushes for region at 1732104815832Disabling writes for close at 1732104815833 (+1 ms)Writing region close event to WAL at 1732104815833Closed at 1732104815833 2024-11-20T12:13:35,834 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0b4c46659583192ce01a652d7688d63d, NAME => 'test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d.', STARTKEY => 'jjj', ENDKEY => 'kkk'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,834 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,834 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing abac5b1bb17e965219c5ca01a8ebc325, disabling compactions & flushes 2024-11-20T12:13:35,834 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:35,835 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:35,835 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. after waiting 0 ms 2024-11-20T12:13:35,835 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:35,835 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:35,835 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for abac5b1bb17e965219c5ca01a8ebc325: Waiting for close lock at 1732104815834Disabling compacts and flushes for region at 1732104815834Disabling writes for close at 1732104815835 (+1 ms)Writing region close event to WAL at 1732104815835Closed at 1732104815835 2024-11-20T12:13:35,836 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c7c42370d5f1944a9469b506921ac97c, NAME => 'test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c.', STARTKEY => 'kkk', ENDKEY => 'lll'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,842 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,842 WARN [IPC Server handler 0 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,842 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,842 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,843 WARN [IPC Server handler 3 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,843 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741846_1022 (size=45) 2024-11-20T12:13:35,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741846_1022 (size=45) 2024-11-20T12:13:35,847 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,847 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing dec6f7c7f05d870bae6d81029b3676e2, disabling compactions & flushes 2024-11-20T12:13:35,847 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:35,847 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:35,848 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. after waiting 0 ms 2024-11-20T12:13:35,848 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:35,848 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:35,849 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for dec6f7c7f05d870bae6d81029b3676e2: Waiting for close lock at 1732104815847Disabling compacts and flushes for region at 1732104815847Disabling writes for close at 1732104815848 (+1 ms)Writing region close event to WAL at 1732104815848Closed at 1732104815848 2024-11-20T12:13:35,850 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 633a6ec91d322a9170f4899c41489180, NAME => 'test,lll,1732104815671.633a6ec91d322a9170f4899c41489180.', STARTKEY => 'lll', ENDKEY => 'mmm'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741847_1023 (size=45) 2024-11-20T12:13:35,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741847_1023 (size=45) 2024-11-20T12:13:35,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741848_1024 (size=45) 2024-11-20T12:13:35,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741848_1024 (size=45) 2024-11-20T12:13:35,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741849_1025 (size=45) 2024-11-20T12:13:35,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741849_1025 (size=45) 2024-11-20T12:13:35,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741849_1025 (size=45) 2024-11-20T12:13:35,885 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,lll,1732104815671.633a6ec91d322a9170f4899c41489180.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,885 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 633a6ec91d322a9170f4899c41489180, disabling compactions & flushes 2024-11-20T12:13:35,886 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:35,886 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:35,886 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. after waiting 0 ms 2024-11-20T12:13:35,886 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:35,886 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:35,886 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 633a6ec91d322a9170f4899c41489180: Waiting for close lock at 1732104815885Disabling compacts and flushes for region at 1732104815885Disabling writes for close at 1732104815886 (+1 ms)Writing region close event to WAL at 1732104815886Closed at 1732104815886 2024-11-20T12:13:35,887 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => f42c0bd7eed4146b0b4502022341fd1d, NAME => 'test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d.', STARTKEY => 'mmm', ENDKEY => 'nnn'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,891 WARN [IPC Server handler 1 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,891 WARN [IPC Server handler 1 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,891 WARN [IPC Server handler 1 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741850_1026 (size=45) 2024-11-20T12:13:35,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741850_1026 (size=45) 2024-11-20T12:13:35,898 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,898 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing f42c0bd7eed4146b0b4502022341fd1d, disabling compactions & flushes 2024-11-20T12:13:35,898 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:35,898 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:35,898 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. after waiting 0 ms 2024-11-20T12:13:35,898 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:35,898 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:35,898 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for f42c0bd7eed4146b0b4502022341fd1d: Waiting for close lock at 1732104815898Disabling compacts and flushes for region at 1732104815898Disabling writes for close at 1732104815898Writing region close event to WAL at 1732104815898Closed at 1732104815898 2024-11-20T12:13:35,899 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 0a3d655f8e46d7b0303a071be25a13da, NAME => 'test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da.', STARTKEY => 'nnn', ENDKEY => 'ooo'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,903 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,903 WARN [IPC Server handler 0 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,903 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741851_1027 (size=45) 2024-11-20T12:13:35,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741851_1027 (size=45) 2024-11-20T12:13:35,921 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,921 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 0a3d655f8e46d7b0303a071be25a13da, disabling compactions & flushes 2024-11-20T12:13:35,921 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:35,921 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:35,921 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. after waiting 0 ms 2024-11-20T12:13:35,921 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:35,921 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:35,921 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 0a3d655f8e46d7b0303a071be25a13da: Waiting for close lock at 1732104815921Disabling compacts and flushes for region at 1732104815921Disabling writes for close at 1732104815921Writing region close event to WAL at 1732104815921Closed at 1732104815921 2024-11-20T12:13:35,922 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 8deafbcc710170143f5159662c491d64, NAME => 'test,ooo,1732104815671.8deafbcc710170143f5159662c491d64.', STARTKEY => 'ooo', ENDKEY => 'ppp'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741852_1028 (size=45) 2024-11-20T12:13:35,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741852_1028 (size=45) 2024-11-20T12:13:35,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741852_1028 (size=45) 2024-11-20T12:13:35,941 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ooo,1732104815671.8deafbcc710170143f5159662c491d64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,941 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 8deafbcc710170143f5159662c491d64, disabling compactions & flushes 2024-11-20T12:13:35,941 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:35,941 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:35,941 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. after waiting 0 ms 2024-11-20T12:13:35,941 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:35,941 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:35,942 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 8deafbcc710170143f5159662c491d64: Waiting for close lock at 1732104815941Disabling compacts and flushes for region at 1732104815941Disabling writes for close at 1732104815941Writing region close event to WAL at 1732104815941Closed at 1732104815941 2024-11-20T12:13:35,943 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 03e3b3cc4355d1bb26b41172ffed46ca, NAME => 'test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca.', STARTKEY => 'ppp', ENDKEY => 'qqq'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741853_1029 (size=45) 2024-11-20T12:13:35,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741853_1029 (size=45) 2024-11-20T12:13:35,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741853_1029 (size=45) 2024-11-20T12:13:35,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 03e3b3cc4355d1bb26b41172ffed46ca, disabling compactions & flushes 2024-11-20T12:13:35,964 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:35,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:35,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. after waiting 0 ms 2024-11-20T12:13:35,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:35,964 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:35,964 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 03e3b3cc4355d1bb26b41172ffed46ca: Waiting for close lock at 1732104815964Disabling compacts and flushes for region at 1732104815964Disabling writes for close at 1732104815964Writing region close event to WAL at 1732104815964Closed at 1732104815964 2024-11-20T12:13:35,965 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => fef27752c5c42f2f9602a38de6f56349, NAME => 'test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349.', STARTKEY => 'qqq', ENDKEY => 'rrr'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,970 WARN [IPC Server handler 1 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,970 WARN [IPC Server handler 1 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,970 WARN [IPC Server handler 1 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741854_1030 (size=45) 2024-11-20T12:13:35,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741854_1030 (size=45) 2024-11-20T12:13:35,977 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,977 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing fef27752c5c42f2f9602a38de6f56349, disabling compactions & flushes 2024-11-20T12:13:35,978 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:35,978 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:35,978 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. after waiting 0 ms 2024-11-20T12:13:35,978 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:35,978 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:35,978 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for fef27752c5c42f2f9602a38de6f56349: Waiting for close lock at 1732104815977Disabling compacts and flushes for region at 1732104815977Disabling writes for close at 1732104815978 (+1 ms)Writing region close event to WAL at 1732104815978Closed at 1732104815978 2024-11-20T12:13:35,979 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 23f5434f0840c36b9d9bb2e81c743305, NAME => 'test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305.', STARTKEY => 'rrr', ENDKEY => 'sss'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,984 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:35,984 WARN [IPC Server handler 0 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:35,984 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:35,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741855_1031 (size=45) 2024-11-20T12:13:35,994 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:35,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 23f5434f0840c36b9d9bb2e81c743305, disabling compactions & flushes 2024-11-20T12:13:35,995 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:35,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:35,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. after waiting 0 ms 2024-11-20T12:13:35,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:35,995 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:35,995 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 23f5434f0840c36b9d9bb2e81c743305: Waiting for close lock at 1732104815995Disabling compacts and flushes for region at 1732104815995Disabling writes for close at 1732104815995Writing region close event to WAL at 1732104815995Closed at 1732104815995 2024-11-20T12:13:35,996 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 45e15b0320018bda387cb884d175aab6, NAME => 'test,sss,1732104815671.45e15b0320018bda387cb884d175aab6.', STARTKEY => 'sss', ENDKEY => 'ttt'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:35,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741855_1031 (size=45) 2024-11-20T12:13:36,001 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:36,001 WARN [IPC Server handler 3 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:36,001 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:36,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741856_1032 (size=45) 2024-11-20T12:13:36,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741856_1032 (size=45) 2024-11-20T12:13:36,010 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,sss,1732104815671.45e15b0320018bda387cb884d175aab6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,011 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 45e15b0320018bda387cb884d175aab6, disabling compactions & flushes 2024-11-20T12:13:36,011 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:36,011 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:36,011 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. after waiting 0 ms 2024-11-20T12:13:36,011 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:36,011 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:36,011 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 45e15b0320018bda387cb884d175aab6: Waiting for close lock at 1732104816011Disabling compacts and flushes for region at 1732104816011Disabling writes for close at 1732104816011Writing region close event to WAL at 1732104816011Closed at 1732104816011 2024-11-20T12:13:36,012 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 312c8b2fa94f26f46326bffa7ae2bcff, NAME => 'test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff.', STARTKEY => 'ttt', ENDKEY => 'uuu'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:36,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:13:36,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741857_1033 (size=45) 2024-11-20T12:13:36,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741857_1033 (size=45) 2024-11-20T12:13:36,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741857_1033 (size=45) 2024-11-20T12:13:36,041 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,042 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 312c8b2fa94f26f46326bffa7ae2bcff, disabling compactions & flushes 2024-11-20T12:13:36,042 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:36,042 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:36,042 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. after waiting 0 ms 2024-11-20T12:13:36,042 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:36,042 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:36,042 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 312c8b2fa94f26f46326bffa7ae2bcff: Waiting for close lock at 1732104816042Disabling compacts and flushes for region at 1732104816042Disabling writes for close at 1732104816042Writing region close event to WAL at 1732104816042Closed at 1732104816042 2024-11-20T12:13:36,043 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(7572): creating {ENCODED => 113951a1eeaf21f60b028a18a0a1a80b, NAME => 'test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b.', STARTKEY => 'uuu', ENDKEY => 'vvv'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:36,056 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:36,056 WARN [IPC Server handler 3 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:36,056 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:36,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741858_1034 (size=45) 2024-11-20T12:13:36,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741858_1034 (size=45) 2024-11-20T12:13:36,264 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,264 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c7c42370d5f1944a9469b506921ac97c, disabling compactions & flushes 2024-11-20T12:13:36,264 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:36,264 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:36,265 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. after waiting 0 ms 2024-11-20T12:13:36,265 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:36,265 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:36,265 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c7c42370d5f1944a9469b506921ac97c: Waiting for close lock at 1732104816264Disabling compacts and flushes for region at 1732104816264Disabling writes for close at 1732104816265 (+1 ms)Writing region close event to WAL at 1732104816265Closed at 1732104816265 2024-11-20T12:13:36,266 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => c2a739d456605f6e8ff10b7be87ca995, NAME => 'test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995.', STARTKEY => 'vvv', ENDKEY => 'www'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:36,270 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:36,270 WARN [IPC Server handler 0 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:36,270 WARN [IPC Server handler 0 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:36,274 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,274 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 0b4c46659583192ce01a652d7688d63d, disabling compactions & flushes 2024-11-20T12:13:36,275 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:36,275 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:36,275 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. after waiting 0 ms 2024-11-20T12:13:36,275 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:36,275 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:36,275 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0b4c46659583192ce01a652d7688d63d: Waiting for close lock at 1732104816274Disabling compacts and flushes for region at 1732104816274Disabling writes for close at 1732104816275 (+1 ms)Writing region close event to WAL at 1732104816275Closed at 1732104816275 2024-11-20T12:13:36,276 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5f89ec3b6d6dbb62bc1c5a7ca523c08c, NAME => 'test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c.', STARTKEY => 'www', ENDKEY => 'xxx'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:36,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741859_1035 (size=45) 2024-11-20T12:13:36,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741859_1035 (size=45) 2024-11-20T12:13:36,278 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,278 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing c2a739d456605f6e8ff10b7be87ca995, disabling compactions & flushes 2024-11-20T12:13:36,278 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:36,278 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:36,278 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. after waiting 0 ms 2024-11-20T12:13:36,278 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:36,279 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:36,279 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for c2a739d456605f6e8ff10b7be87ca995: Waiting for close lock at 1732104816278Disabling compacts and flushes for region at 1732104816278Disabling writes for close at 1732104816278Writing region close event to WAL at 1732104816279 (+1 ms)Closed at 1732104816279 2024-11-20T12:13:36,280 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 2186bb28c010fe9bd50ae06c405d542b, NAME => 'test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b.', STARTKEY => 'xxx', ENDKEY => 'yyy'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:36,286 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:36,286 WARN [IPC Server handler 3 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:36,286 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:36,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741860_1036 (size=45) 2024-11-20T12:13:36,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741860_1036 (size=45) 2024-11-20T12:13:36,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741860_1036 (size=45) 2024-11-20T12:13:36,293 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,293 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 5f89ec3b6d6dbb62bc1c5a7ca523c08c, disabling compactions & flushes 2024-11-20T12:13:36,293 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:36,294 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:36,294 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. after waiting 0 ms 2024-11-20T12:13:36,294 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:36,294 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:36,294 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5f89ec3b6d6dbb62bc1c5a7ca523c08c: Waiting for close lock at 1732104816293Disabling compacts and flushes for region at 1732104816293Disabling writes for close at 1732104816294 (+1 ms)Writing region close event to WAL at 1732104816294Closed at 1732104816294 2024-11-20T12:13:36,295 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 8978fd254a3dd3cd4570b36a1c18b8cb, NAME => 'test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb.', STARTKEY => 'yyy', ENDKEY => 'zzz'}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:36,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741861_1037 (size=45) 2024-11-20T12:13:36,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741861_1037 (size=45) 2024-11-20T12:13:36,299 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,299 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 2186bb28c010fe9bd50ae06c405d542b, disabling compactions & flushes 2024-11-20T12:13:36,299 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:36,299 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:36,299 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. after waiting 0 ms 2024-11-20T12:13:36,299 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:36,299 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:36,299 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 2186bb28c010fe9bd50ae06c405d542b: Waiting for close lock at 1732104816299Disabling compacts and flushes for region at 1732104816299Disabling writes for close at 1732104816299Writing region close event to WAL at 1732104816299Closed at 1732104816299 2024-11-20T12:13:36,300 WARN [IPC Server handler 4 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:36,300 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(7572): creating {ENCODED => 366d4b256c89d7860c4d7747ae8d7cc6, NAME => 'test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6.', STARTKEY => 'zzz', ENDKEY => ''}, tableDescriptor='test', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad 2024-11-20T12:13:36,300 WARN [IPC Server handler 4 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:36,300 WARN [IPC Server handler 4 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:36,305 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-11-20T12:13:36,305 WARN [IPC Server handler 3 on default port 42641 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-11-20T12:13:36,306 WARN [IPC Server handler 3 on default port 42641 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-11-20T12:13:36,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741862_1038 (size=45) 2024-11-20T12:13:36,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741862_1038 (size=45) 2024-11-20T12:13:36,308 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(898): Instantiated test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,308 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1722): Closing 8978fd254a3dd3cd4570b36a1c18b8cb, disabling compactions & flushes 2024-11-20T12:13:36,308 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1755): Closing region test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:36,308 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:36,308 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. after waiting 0 ms 2024-11-20T12:13:36,308 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:36,308 INFO [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1973): Closed test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:36,308 DEBUG [RegionOpenAndInit-test-pool-0 {}] regionserver.HRegion(1676): Region close journal for 8978fd254a3dd3cd4570b36a1c18b8cb: Waiting for close lock at 1732104816308Disabling compacts and flushes for region at 1732104816308Disabling writes for close at 1732104816308Writing region close event to WAL at 1732104816308Closed at 1732104816308 2024-11-20T12:13:36,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741863_1039 (size=42) 2024-11-20T12:13:36,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741863_1039 (size=42) 2024-11-20T12:13:36,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(898): Instantiated test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1722): Closing 366d4b256c89d7860c4d7747ae8d7cc6, disabling compactions & flushes 2024-11-20T12:13:36,313 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1755): Closing region test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:36,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:36,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. after waiting 0 ms 2024-11-20T12:13:36,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:36,313 INFO [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1973): Closed test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:36,313 DEBUG [RegionOpenAndInit-test-pool-2 {}] regionserver.HRegion(1676): Region close journal for 366d4b256c89d7860c4d7747ae8d7cc6: Waiting for close lock at 1732104816313Disabling compacts and flushes for region at 1732104816313Disabling writes for close at 1732104816313Writing region close event to WAL at 1732104816313Closed at 1732104816313 2024-11-20T12:13:36,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:13:36,468 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(898): Instantiated test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,468 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1722): Closing 113951a1eeaf21f60b028a18a0a1a80b, disabling compactions & flushes 2024-11-20T12:13:36,468 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1755): Closing region test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:36,468 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:36,468 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. after waiting 0 ms 2024-11-20T12:13:36,468 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:36,468 INFO [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1973): Closed test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:36,468 DEBUG [RegionOpenAndInit-test-pool-1 {}] regionserver.HRegion(1676): Region close journal for 113951a1eeaf21f60b028a18a0a1a80b: Waiting for close lock at 1732104816468Disabling compacts and flushes for region at 1732104816468Disabling writes for close at 1732104816468Writing region close event to WAL at 1732104816468Closed at 1732104816468 2024-11-20T12:13:36,471 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ADD_TO_META 2024-11-20T12:13:36,476 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,476 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,,1732104815671.99f9c663ffb10623454992ca39cafe6e.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,476 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,476 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,476 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,477 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,477 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,477 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,477 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,477 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,lll,1732104815671.633a6ec91d322a9170f4899c41489180.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,491 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,491 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,492 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ooo,1732104815671.8deafbcc710170143f5159662c491d64.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,492 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,492 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,492 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,492 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,sss,1732104815671.45e15b0320018bda387cb884d175aab6.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,492 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,492 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,493 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,493 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,493 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,493 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,493 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,493 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,493 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b.","families":{"info":[{"qualifier":"regioninfo","vlen":44,"tag":[],"timestamp":"1732104816471"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732104816471"}]},"ts":"1732104816471"} 2024-11-20T12:13:36,549 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 26 regions to meta. 2024-11-20T12:13:36,552 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-20T12:13:36,556 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104816552"}]},"ts":"1732104816552"} 2024-11-20T12:13:36,561 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLING in hbase:meta 2024-11-20T12:13:36,562 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(204): Hosts are {843d83d51ba0=0} racks are {/default-rack=0} 2024-11-20T12:13:36,581 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T12:13:36,581 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T12:13:36,581 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-20T12:13:36,581 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T12:13:36,581 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T12:13:36,581 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T12:13:36,581 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T12:13:36,581 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T12:13:36,581 INFO [PEWorker-1 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T12:13:36,581 DEBUG [PEWorker-1 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T12:13:36,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN}, {pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23c5a5a22a2d09813d1ff472bb76f407, ASSIGN}, {pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ae055eeceea830ff461b6b4dc158461e, ASSIGN}, {pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b2ba9c7430b8a524b3966df315ad45c, ASSIGN}, {pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN}, {pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=325e5f934bae06dfb2390407a9e9726f, ASSIGN}, {pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN}, {pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6171c0ffde5383e6eb4635a4c76e3ad1, ASSIGN}, {pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dec6f7c7f05d870bae6d81029b3676e2, ASSIGN}, {pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN}, {pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c7c42370d5f1944a9469b506921ac97c, ASSIGN}, {pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=633a6ec91d322a9170f4899c41489180, ASSIGN}, {pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN}, {pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0a3d655f8e46d7b0303a071be25a13da, ASSIGN}, {pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8deafbcc710170143f5159662c491d64, ASSIGN}, {pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN}, {pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fef27752c5c42f2f9602a38de6f56349, ASSIGN}, {pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23f5434f0840c36b9d9bb2e81c743305, ASSIGN}, {pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN}, {pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=312c8b2fa94f26f46326bffa7ae2bcff, ASSIGN}, {pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, ASSIGN}, {pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c2a739d456605f6e8ff10b7be87ca995, ASSIGN}, {pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, ASSIGN}, {pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN}, {pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8978fd254a3dd3cd4570b36a1c18b8cb, ASSIGN}, {pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=366d4b256c89d7860c4d7747ae8d7cc6, ASSIGN}] 2024-11-20T12:13:36,601 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=366d4b256c89d7860c4d7747ae8d7cc6, ASSIGN 2024-11-20T12:13:36,603 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=366d4b256c89d7860c4d7747ae8d7cc6, ASSIGN; state=OFFLINE, location=843d83d51ba0,44663,1732104813203; forceNewPlan=false, retain=false 2024-11-20T12:13:36,605 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8978fd254a3dd3cd4570b36a1c18b8cb, ASSIGN 2024-11-20T12:13:36,606 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8978fd254a3dd3cd4570b36a1c18b8cb, ASSIGN; state=OFFLINE, location=843d83d51ba0,38659,1732104813354; forceNewPlan=false, retain=false 2024-11-20T12:13:36,608 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:36,609 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN; state=OFFLINE, location=843d83d51ba0,35591,1732104813307; forceNewPlan=false, retain=false 2024-11-20T12:13:36,611 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, ASSIGN 2024-11-20T12:13:36,613 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, ASSIGN; state=OFFLINE, location=843d83d51ba0,44663,1732104813203; forceNewPlan=false, retain=false 2024-11-20T12:13:36,614 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c2a739d456605f6e8ff10b7be87ca995, ASSIGN 2024-11-20T12:13:36,616 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c2a739d456605f6e8ff10b7be87ca995, ASSIGN; state=OFFLINE, location=843d83d51ba0,38659,1732104813354; forceNewPlan=false, retain=false 2024-11-20T12:13:36,617 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, ASSIGN 2024-11-20T12:13:36,618 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, ASSIGN; state=OFFLINE, location=843d83d51ba0,35591,1732104813307; forceNewPlan=false, retain=false 2024-11-20T12:13:36,620 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=312c8b2fa94f26f46326bffa7ae2bcff, ASSIGN 2024-11-20T12:13:36,621 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=312c8b2fa94f26f46326bffa7ae2bcff, ASSIGN; state=OFFLINE, location=843d83d51ba0,38659,1732104813354; forceNewPlan=false, retain=false 2024-11-20T12:13:36,623 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:36,624 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN; state=OFFLINE, location=843d83d51ba0,35591,1732104813307; forceNewPlan=false, retain=false 2024-11-20T12:13:36,625 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23f5434f0840c36b9d9bb2e81c743305, ASSIGN 2024-11-20T12:13:36,633 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=23f5434f0840c36b9d9bb2e81c743305, ASSIGN; state=OFFLINE, location=843d83d51ba0,44663,1732104813203; forceNewPlan=false, retain=false 2024-11-20T12:13:36,634 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=fef27752c5c42f2f9602a38de6f56349, ASSIGN 2024-11-20T12:13:36,636 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=fef27752c5c42f2f9602a38de6f56349, ASSIGN; state=OFFLINE, location=843d83d51ba0,38659,1732104813354; forceNewPlan=false, retain=false 2024-11-20T12:13:36,637 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:36,639 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN; state=OFFLINE, location=843d83d51ba0,35591,1732104813307; forceNewPlan=false, retain=false 2024-11-20T12:13:36,640 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8deafbcc710170143f5159662c491d64, ASSIGN 2024-11-20T12:13:36,642 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8deafbcc710170143f5159662c491d64, ASSIGN; state=OFFLINE, location=843d83d51ba0,44663,1732104813203; forceNewPlan=false, retain=false 2024-11-20T12:13:36,643 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0a3d655f8e46d7b0303a071be25a13da, ASSIGN 2024-11-20T12:13:36,645 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0a3d655f8e46d7b0303a071be25a13da, ASSIGN; state=OFFLINE, location=843d83d51ba0,38659,1732104813354; forceNewPlan=false, retain=false 2024-11-20T12:13:36,646 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:36,647 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN; state=OFFLINE, location=843d83d51ba0,35591,1732104813307; forceNewPlan=false, retain=false 2024-11-20T12:13:36,649 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=633a6ec91d322a9170f4899c41489180, ASSIGN 2024-11-20T12:13:36,650 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=633a6ec91d322a9170f4899c41489180, ASSIGN; state=OFFLINE, location=843d83d51ba0,44663,1732104813203; forceNewPlan=false, retain=false 2024-11-20T12:13:36,651 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=c7c42370d5f1944a9469b506921ac97c, ASSIGN 2024-11-20T12:13:36,652 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=c7c42370d5f1944a9469b506921ac97c, ASSIGN; state=OFFLINE, location=843d83d51ba0,44663,1732104813203; forceNewPlan=false, retain=false 2024-11-20T12:13:36,654 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:36,655 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN; state=OFFLINE, location=843d83d51ba0,35591,1732104813307; forceNewPlan=false, retain=false 2024-11-20T12:13:36,656 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=dec6f7c7f05d870bae6d81029b3676e2, ASSIGN 2024-11-20T12:13:36,657 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=dec6f7c7f05d870bae6d81029b3676e2, ASSIGN; state=OFFLINE, location=843d83d51ba0,38659,1732104813354; forceNewPlan=false, retain=false 2024-11-20T12:13:36,658 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=6171c0ffde5383e6eb4635a4c76e3ad1, ASSIGN 2024-11-20T12:13:36,659 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=6171c0ffde5383e6eb4635a4c76e3ad1, ASSIGN; state=OFFLINE, location=843d83d51ba0,44663,1732104813203; forceNewPlan=false, retain=false 2024-11-20T12:13:36,661 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:36,662 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN; state=OFFLINE, location=843d83d51ba0,35591,1732104813307; forceNewPlan=false, retain=false 2024-11-20T12:13:36,663 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=325e5f934bae06dfb2390407a9e9726f, ASSIGN 2024-11-20T12:13:36,665 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=325e5f934bae06dfb2390407a9e9726f, ASSIGN; state=OFFLINE, location=843d83d51ba0,38659,1732104813354; forceNewPlan=false, retain=false 2024-11-20T12:13:36,666 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:36,667 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN; state=OFFLINE, location=843d83d51ba0,35591,1732104813307; forceNewPlan=false, retain=false 2024-11-20T12:13:36,668 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=9b2ba9c7430b8a524b3966df315ad45c, ASSIGN 2024-11-20T12:13:36,669 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=9b2ba9c7430b8a524b3966df315ad45c, ASSIGN; state=OFFLINE, location=843d83d51ba0,44663,1732104813203; forceNewPlan=false, retain=false 2024-11-20T12:13:36,671 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=ae055eeceea830ff461b6b4dc158461e, ASSIGN 2024-11-20T12:13:36,672 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=ae055eeceea830ff461b6b4dc158461e, ASSIGN; state=OFFLINE, location=843d83d51ba0,44663,1732104813203; forceNewPlan=false, retain=false 2024-11-20T12:13:36,673 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=23c5a5a22a2d09813d1ff472bb76f407, ASSIGN 2024-11-20T12:13:36,674 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=23c5a5a22a2d09813d1ff472bb76f407, ASSIGN; state=OFFLINE, location=843d83d51ba0,38659,1732104813354; forceNewPlan=false, retain=false 2024-11-20T12:13:36,675 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:36,676 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN; state=OFFLINE, location=843d83d51ba0,35591,1732104813307; forceNewPlan=false, retain=false 2024-11-20T12:13:36,758 INFO [843d83d51ba0:39157 {}] balancer.BaseLoadBalancer(388): Reassigned 26 regions. 26 retained the pre-restart assignment. 2024-11-20T12:13:36,760 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:36,761 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:36,762 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=23f5434f0840c36b9d9bb2e81c743305, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:36,764 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=8deafbcc710170143f5159662c491d64, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:36,765 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=633a6ec91d322a9170f4899c41489180, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:36,765 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=366d4b256c89d7860c4d7747ae8d7cc6, ASSIGN because future has completed 2024-11-20T12:13:36,766 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=c7c42370d5f1944a9469b506921ac97c, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:36,767 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE, hasLock=false; OpenRegionProcedure 366d4b256c89d7860c4d7747ae8d7cc6, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:36,768 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, ASSIGN because future has completed 2024-11-20T12:13:36,770 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=23f5434f0840c36b9d9bb2e81c743305, ASSIGN because future has completed 2024-11-20T12:13:36,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=19, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8deafbcc710170143f5159662c491d64, ASSIGN because future has completed 2024-11-20T12:13:36,773 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8deafbcc710170143f5159662c491d64, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:36,774 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=633a6ec91d322a9170f4899c41489180, ASSIGN because future has completed 2024-11-20T12:13:36,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=15, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c7c42370d5f1944a9469b506921ac97c, ASSIGN because future has completed 2024-11-20T12:13:36,779 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7c42370d5f1944a9469b506921ac97c, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:36,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 633a6ec91d322a9170f4899c41489180, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:36,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f5434f0840c36b9d9bb2e81c743305, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:36,794 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f89ec3b6d6dbb62bc1c5a7ca523c08c, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:36,798 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:36,799 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:36,800 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ae055eeceea830ff461b6b4dc158461e, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:36,801 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:36,802 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c2a739d456605f6e8ff10b7be87ca995, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:36,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=6171c0ffde5383e6eb4635a4c76e3ad1, ASSIGN because future has completed 2024-11-20T12:13:36,803 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:36,803 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=fef27752c5c42f2f9602a38de6f56349, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:36,804 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=0a3d655f8e46d7b0303a071be25a13da, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:36,805 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:36,806 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=325e5f934bae06dfb2390407a9e9726f, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:36,807 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:36,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6171c0ffde5383e6eb4635a4c76e3ad1, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:36,810 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=9b2ba9c7430b8a524b3966df315ad45c, ASSIGN because future has completed 2024-11-20T12:13:36,814 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=ae055eeceea830ff461b6b4dc158461e, ASSIGN because future has completed 2024-11-20T12:13:36,816 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8978fd254a3dd3cd4570b36a1c18b8cb, ASSIGN because future has completed 2024-11-20T12:13:36,817 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8978fd254a3dd3cd4570b36a1c18b8cb, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:36,818 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=c2a739d456605f6e8ff10b7be87ca995, ASSIGN because future has completed 2024-11-20T12:13:36,826 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=312c8b2fa94f26f46326bffa7ae2bcff, ASSIGN because future has completed 2024-11-20T12:13:36,828 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=fef27752c5c42f2f9602a38de6f56349, ASSIGN because future has completed 2024-11-20T12:13:36,830 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=18, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0a3d655f8e46d7b0303a071be25a13da, ASSIGN because future has completed 2024-11-20T12:13:36,831 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a3d655f8e46d7b0303a071be25a13da, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:36,834 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=13, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=dec6f7c7f05d870bae6d81029b3676e2, ASSIGN because future has completed 2024-11-20T12:13:36,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=325e5f934bae06dfb2390407a9e9726f, ASSIGN because future has completed 2024-11-20T12:13:36,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:13:36,839 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=23c5a5a22a2d09813d1ff472bb76f407, ASSIGN because future has completed 2024-11-20T12:13:36,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23c5a5a22a2d09813d1ff472bb76f407, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:36,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 325e5f934bae06dfb2390407a9e9726f, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:36,860 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=42, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure dec6f7c7f05d870bae6d81029b3676e2, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:36,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure fef27752c5c42f2f9602a38de6f56349, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:36,879 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 312c8b2fa94f26f46326bffa7ae2bcff, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:36,889 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2a739d456605f6e8ff10b7be87ca995, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:36,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure ae055eeceea830ff461b6b4dc158461e, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:36,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2ba9c7430b8a524b3966df315ad45c, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:36,912 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:36,913 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPENING, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:36,914 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=45e15b0320018bda387cb884d175aab6, regionState=OPENING, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:36,915 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:36,916 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:36,916 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:36,917 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:36,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN because future has completed 2024-11-20T12:13:36,918 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,35591,1732104813307}] 2024-11-20T12:13:36,920 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, ASSIGN because future has completed 2024-11-20T12:13:36,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN because future has completed 2024-11-20T12:13:36,923 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T12:13:36,924 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN because future has completed 2024-11-20T12:13:36,926 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54289, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T12:13:36,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN because future has completed 2024-11-20T12:13:36,929 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,35591,1732104813307}] 2024-11-20T12:13:36,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=14, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN because future has completed 2024-11-20T12:13:36,932 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN because future has completed 2024-11-20T12:13:36,936 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:36,937 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,35591,1732104813307}] 2024-11-20T12:13:36,937 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => c7c42370d5f1944a9469b506921ac97c, NAME => 'test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c.', STARTKEY => 'kkk', ENDKEY => 'lll'} 2024-11-20T12:13:36,937 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:36,938 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,938 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:36,938 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:36,944 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b4c46659583192ce01a652d7688d63d, server=843d83d51ba0,35591,1732104813307}] 2024-11-20T12:13:36,944 INFO [StoreOpener-c7c42370d5f1944a9469b506921ac97c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:36,948 INFO [StoreOpener-c7c42370d5f1944a9469b506921ac97c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c7c42370d5f1944a9469b506921ac97c columnFamilyName family 2024-11-20T12:13:36,948 DEBUG [StoreOpener-c7c42370d5f1944a9469b506921ac97c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:36,950 INFO [StoreOpener-c7c42370d5f1944a9469b506921ac97c-1 {}] regionserver.HStore(327): Store=c7c42370d5f1944a9469b506921ac97c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:36,950 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:36,952 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:36,953 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:36,954 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,35591,1732104813307}] 2024-11-20T12:13:36,954 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:36,954 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:36,957 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:36,961 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/c7c42370d5f1944a9469b506921ac97c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:36,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,35591,1732104813307}] 2024-11-20T12:13:36,962 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened c7c42370d5f1944a9469b506921ac97c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58733931, jitterRate=-0.12479622662067413}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:36,962 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:36,963 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for c7c42370d5f1944a9469b506921ac97c: Running coprocessor pre-open hook at 1732104816938Writing region info on filesystem at 1732104816938Initializing all the Stores at 1732104816940 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104816940Cleaning up temporary data from old regions at 1732104816954 (+14 ms)Running coprocessor post-open hooks at 1732104816962 (+8 ms)Region opened successfully at 1732104816963 (+1 ms) 2024-11-20T12:13:36,965 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c., pid=33, masterSystemTime=1732104816923 2024-11-20T12:13:36,969 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:36,969 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:36,969 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(132): Open test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:36,970 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=15 updating hbase:meta row=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:36,970 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7752): Opening region: {ENCODED => 23f5434f0840c36b9d9bb2e81c743305, NAME => 'test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305.', STARTKEY => 'rrr', ENDKEY => 'sss'} 2024-11-20T12:13:36,971 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:36,971 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(898): Instantiated test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,971 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7794): checking encryption for 23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:36,971 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(7797): checking classloading for 23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:36,973 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 113951a1eeaf21f60b028a18a0a1a80b, server=843d83d51ba0,35591,1732104813307}] 2024-11-20T12:13:36,974 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=15, state=RUNNABLE, hasLock=false; OpenRegionProcedure c7c42370d5f1944a9469b506921ac97c, server=843d83d51ba0,44663,1732104813203 because future has completed 2024-11-20T12:13:36,975 INFO [StoreOpener-23f5434f0840c36b9d9bb2e81c743305-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:36,978 DEBUG [RSProcedureDispatcher-pool-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-20T12:13:36,978 INFO [StoreOpener-23f5434f0840c36b9d9bb2e81c743305-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23f5434f0840c36b9d9bb2e81c743305 columnFamilyName family 2024-11-20T12:13:36,979 DEBUG [StoreOpener-23f5434f0840c36b9d9bb2e81c743305-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:36,980 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56291, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-20T12:13:36,980 INFO [StoreOpener-23f5434f0840c36b9d9bb2e81c743305-1 {}] regionserver.HStore(327): Store=23f5434f0840c36b9d9bb2e81c743305/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:36,980 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1038): replaying wal for 23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:36,982 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:36,985 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:36,986 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1048): stopping wal replay for 23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:36,986 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1060): Cleaning up temporary data for 23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:36,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=15 2024-11-20T12:13:36,991 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=15, state=SUCCESS, hasLock=false; OpenRegionProcedure c7c42370d5f1944a9469b506921ac97c, server=843d83d51ba0,44663,1732104813203 in 205 msec 2024-11-20T12:13:36,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c7c42370d5f1944a9469b506921ac97c, ASSIGN in 401 msec 2024-11-20T12:13:36,996 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:36,996 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(132): Open test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:36,997 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7752): Opening region: {ENCODED => c2a739d456605f6e8ff10b7be87ca995, NAME => 'test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995.', STARTKEY => 'vvv', ENDKEY => 'www'} 2024-11-20T12:13:36,997 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:36,997 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:36,997 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(898): Instantiated test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:36,998 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7794): checking encryption for c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:36,998 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(7797): checking classloading for c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:36,999 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1093): writing seq id for 23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:37,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN because future has completed 2024-11-20T12:13:37,001 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8448ac25ea038ac8c63c9da8906d8673, server=843d83d51ba0,35591,1732104813307}] 2024-11-20T12:13:37,002 INFO [StoreOpener-c2a739d456605f6e8ff10b7be87ca995-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:37,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN because future has completed 2024-11-20T12:13:37,005 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/23f5434f0840c36b9d9bb2e81c743305/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,005 INFO [StoreOpener-c2a739d456605f6e8ff10b7be87ca995-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region c2a739d456605f6e8ff10b7be87ca995 columnFamilyName family 2024-11-20T12:13:37,005 DEBUG [StoreOpener-c2a739d456605f6e8ff10b7be87ca995-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,006 INFO [StoreOpener-c2a739d456605f6e8ff10b7be87ca995-1 {}] regionserver.HStore(327): Store=c2a739d456605f6e8ff10b7be87ca995/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,006 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1114): Opened 23f5434f0840c36b9d9bb2e81c743305; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72278005, jitterRate=0.07702620327472687}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,006 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:37,006 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegion(1006): Region open journal for 23f5434f0840c36b9d9bb2e81c743305: Running coprocessor pre-open hook at 1732104816972Writing region info on filesystem at 1732104816972Initializing all the Stores at 1732104816974 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104816974Cleaning up temporary data from old regions at 1732104816986 (+12 ms)Running coprocessor post-open hooks at 1732104817006 (+20 ms)Region opened successfully at 1732104817006 2024-11-20T12:13:37,007 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1038): replaying wal for c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:37,009 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2236): Post open deploy tasks for test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305., pid=35, masterSystemTime=1732104816923 2024-11-20T12:13:37,009 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:37,010 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:37,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,35591,1732104813307}] 2024-11-20T12:13:37,010 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1048): stopping wal replay for c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:37,011 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1060): Cleaning up temporary data for c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:37,013 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] regionserver.HRegionServer(2266): Finished post open deploy task for test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:37,013 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=35}] handler.AssignRegionHandler(153): Opened test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:37,013 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(132): Open test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:37,013 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7752): Opening region: {ENCODED => 8deafbcc710170143f5159662c491d64, NAME => 'test,ooo,1732104815671.8deafbcc710170143f5159662c491d64.', STARTKEY => 'ooo', ENDKEY => 'ppp'} 2024-11-20T12:13:37,014 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=22 updating hbase:meta row=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:37,014 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8deafbcc710170143f5159662c491d64 2024-11-20T12:13:37,014 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(898): Instantiated test,ooo,1732104815671.8deafbcc710170143f5159662c491d64.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,014 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7794): checking encryption for 8deafbcc710170143f5159662c491d64 2024-11-20T12:13:37,014 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1093): writing seq id for c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:37,014 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(7797): checking classloading for 8deafbcc710170143f5159662c491d64 2024-11-20T12:13:37,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=22, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23f5434f0840c36b9d9bb2e81c743305, server=843d83d51ba0,44663,1732104813203 because future has completed 2024-11-20T12:13:37,018 INFO [StoreOpener-8deafbcc710170143f5159662c491d64-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8deafbcc710170143f5159662c491d64 2024-11-20T12:13:37,022 INFO [StoreOpener-8deafbcc710170143f5159662c491d64-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8deafbcc710170143f5159662c491d64 columnFamilyName family 2024-11-20T12:13:37,022 DEBUG [StoreOpener-8deafbcc710170143f5159662c491d64-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,023 INFO [StoreOpener-8deafbcc710170143f5159662c491d64-1 {}] regionserver.HStore(327): Store=8deafbcc710170143f5159662c491d64/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,023 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1038): replaying wal for 8deafbcc710170143f5159662c491d64 2024-11-20T12:13:37,025 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8deafbcc710170143f5159662c491d64 2024-11-20T12:13:37,025 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/c2a739d456605f6e8ff10b7be87ca995/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=22 2024-11-20T12:13:37,026 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=22, state=SUCCESS, hasLock=false; OpenRegionProcedure 23f5434f0840c36b9d9bb2e81c743305, server=843d83d51ba0,44663,1732104813203 in 234 msec 2024-11-20T12:13:37,026 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8deafbcc710170143f5159662c491d64 2024-11-20T12:13:37,026 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1048): stopping wal replay for 8deafbcc710170143f5159662c491d64 2024-11-20T12:13:37,026 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1060): Cleaning up temporary data for 8deafbcc710170143f5159662c491d64 2024-11-20T12:13:37,026 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1114): Opened c2a739d456605f6e8ff10b7be87ca995; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69526015, jitterRate=0.036018356680870056}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,027 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1122): Running coprocessor post-open hooks for c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:37,027 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegion(1006): Region open journal for c2a739d456605f6e8ff10b7be87ca995: Running coprocessor pre-open hook at 1732104816998Writing region info on filesystem at 1732104816998Initializing all the Stores at 1732104817000 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817000Cleaning up temporary data from old regions at 1732104817011 (+11 ms)Running coprocessor post-open hooks at 1732104817027 (+16 ms)Region opened successfully at 1732104817027 2024-11-20T12:13:37,028 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2236): Post open deploy tasks for test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995., pid=45, masterSystemTime=1732104816977 2024-11-20T12:13:37,030 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23f5434f0840c36b9d9bb2e81c743305, ASSIGN in 436 msec 2024-11-20T12:13:37,031 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1093): writing seq id for 8deafbcc710170143f5159662c491d64 2024-11-20T12:13:37,033 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=26 updating hbase:meta row=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:37,035 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] regionserver.HRegionServer(2266): Finished post open deploy task for test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:37,036 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=45}] handler.AssignRegionHandler(153): Opened test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:37,036 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:37,036 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 325e5f934bae06dfb2390407a9e9726f, NAME => 'test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f.', STARTKEY => 'fff', ENDKEY => 'ggg'} 2024-11-20T12:13:37,037 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:37,037 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,037 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:37,037 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:37,039 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=45, ppid=26, state=RUNNABLE, hasLock=false; OpenRegionProcedure c2a739d456605f6e8ff10b7be87ca995, server=843d83d51ba0,38659,1732104813354 because future has completed 2024-11-20T12:13:37,039 INFO [StoreOpener-325e5f934bae06dfb2390407a9e9726f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:37,039 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8deafbcc710170143f5159662c491d64/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,040 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1114): Opened 8deafbcc710170143f5159662c491d64; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65215551, jitterRate=-0.028212562203407288}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,041 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8deafbcc710170143f5159662c491d64 2024-11-20T12:13:37,041 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegion(1006): Region open journal for 8deafbcc710170143f5159662c491d64: Running coprocessor pre-open hook at 1732104817014Writing region info on filesystem at 1732104817014Initializing all the Stores at 1732104817017 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817017Cleaning up temporary data from old regions at 1732104817027 (+10 ms)Running coprocessor post-open hooks at 1732104817041 (+14 ms)Region opened successfully at 1732104817041 2024-11-20T12:13:37,043 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ooo,1732104815671.8deafbcc710170143f5159662c491d64., pid=32, masterSystemTime=1732104816923 2024-11-20T12:13:37,043 INFO [StoreOpener-325e5f934bae06dfb2390407a9e9726f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 325e5f934bae06dfb2390407a9e9726f columnFamilyName family 2024-11-20T12:13:37,043 DEBUG [StoreOpener-325e5f934bae06dfb2390407a9e9726f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,044 INFO [StoreOpener-325e5f934bae06dfb2390407a9e9726f-1 {}] regionserver.HStore(327): Store=325e5f934bae06dfb2390407a9e9726f/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,045 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:37,046 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:37,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=26 2024-11-20T12:13:37,046 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=26, state=SUCCESS, hasLock=false; OpenRegionProcedure c2a739d456605f6e8ff10b7be87ca995, server=843d83d51ba0,38659,1732104813354 in 153 msec 2024-11-20T12:13:37,047 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:37,047 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:37,047 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=32}] handler.AssignRegionHandler(153): Opened test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:37,047 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:37,048 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:37,048 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:37,048 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 633a6ec91d322a9170f4899c41489180, NAME => 'test,lll,1732104815671.633a6ec91d322a9170f4899c41489180.', STARTKEY => 'lll', ENDKEY => 'mmm'} 2024-11-20T12:13:37,048 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:37,048 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=19 updating hbase:meta row=8deafbcc710170143f5159662c491d64, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:37,048 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated test,lll,1732104815671.633a6ec91d322a9170f4899c41489180.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,049 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:37,049 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:37,051 INFO [StoreOpener-633a6ec91d322a9170f4899c41489180-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:37,053 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:37,053 INFO [StoreOpener-633a6ec91d322a9170f4899c41489180-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 633a6ec91d322a9170f4899c41489180 columnFamilyName family 2024-11-20T12:13:37,054 DEBUG [StoreOpener-633a6ec91d322a9170f4899c41489180-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,054 INFO [StoreOpener-633a6ec91d322a9170f4899c41489180-1 {}] regionserver.HStore(327): Store=633a6ec91d322a9170f4899c41489180/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,054 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=19, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8deafbcc710170143f5159662c491d64, server=843d83d51ba0,44663,1732104813203 because future has completed 2024-11-20T12:13:37,055 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:37,056 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:37,058 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/325e5f934bae06dfb2390407a9e9726f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,058 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:37,059 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 325e5f934bae06dfb2390407a9e9726f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59948678, jitterRate=-0.10669508576393127}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,059 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:37,059 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 325e5f934bae06dfb2390407a9e9726f: Running coprocessor pre-open hook at 1732104817037Writing region info on filesystem at 1732104817037Initializing all the Stores at 1732104817039 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817039Cleaning up temporary data from old regions at 1732104817048 (+9 ms)Running coprocessor post-open hooks at 1732104817059 (+11 ms)Region opened successfully at 1732104817059 2024-11-20T12:13:37,059 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:37,060 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:37,060 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f., pid=41, masterSystemTime=1732104816977 2024-11-20T12:13:37,061 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=c2a739d456605f6e8ff10b7be87ca995, ASSIGN in 466 msec 2024-11-20T12:13:37,064 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:37,065 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:37,065 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:37,065 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(132): Open test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:37,066 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7752): Opening region: {ENCODED => fef27752c5c42f2f9602a38de6f56349, NAME => 'test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349.', STARTKEY => 'qqq', ENDKEY => 'rrr'} 2024-11-20T12:13:37,066 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:37,066 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(898): Instantiated test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,067 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7794): checking encryption for fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:37,067 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(7797): checking classloading for fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:37,069 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/633a6ec91d322a9170f4899c41489180/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,070 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 633a6ec91d322a9170f4899c41489180; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59295454, jitterRate=-0.11642888188362122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,070 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:37,070 INFO [StoreOpener-fef27752c5c42f2f9602a38de6f56349-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:37,070 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=19 2024-11-20T12:13:37,070 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 633a6ec91d322a9170f4899c41489180: Running coprocessor pre-open hook at 1732104817049Writing region info on filesystem at 1732104817049Initializing all the Stores at 1732104817050 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817050Cleaning up temporary data from old regions at 1732104817060 (+10 ms)Running coprocessor post-open hooks at 1732104817070 (+10 ms)Region opened successfully at 1732104817070 2024-11-20T12:13:37,071 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=19, state=SUCCESS, hasLock=false; OpenRegionProcedure 8deafbcc710170143f5159662c491d64, server=843d83d51ba0,44663,1732104813203 in 291 msec 2024-11-20T12:13:37,072 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for test,lll,1732104815671.633a6ec91d322a9170f4899c41489180., pid=34, masterSystemTime=1732104816923 2024-11-20T12:13:37,074 INFO [StoreOpener-fef27752c5c42f2f9602a38de6f56349-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fef27752c5c42f2f9602a38de6f56349 columnFamilyName family 2024-11-20T12:13:37,074 DEBUG [StoreOpener-fef27752c5c42f2f9602a38de6f56349-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,075 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8deafbcc710170143f5159662c491d64, ASSIGN in 481 msec 2024-11-20T12:13:37,076 INFO [StoreOpener-fef27752c5c42f2f9602a38de6f56349-1 {}] regionserver.HStore(327): Store=fef27752c5c42f2f9602a38de6f56349/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,076 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1038): replaying wal for fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:37,078 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:37,078 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:37,079 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:37,079 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:37,079 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:37,079 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f89ec3b6d6dbb62bc1c5a7ca523c08c, NAME => 'test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c.', STARTKEY => 'www', ENDKEY => 'xxx'} 2024-11-20T12:13:37,080 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:37,080 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,080 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:37,080 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:37,081 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:37,081 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1048): stopping wal replay for fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:37,082 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1060): Cleaning up temporary data for fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:37,083 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=633a6ec91d322a9170f4899c41489180, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:37,083 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(132): Open test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:37,083 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7752): Opening region: {ENCODED => 03e3b3cc4355d1bb26b41172ffed46ca, NAME => 'test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-20T12:13:37,083 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:37,084 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(898): Instantiated test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,084 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7794): checking encryption for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:37,084 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(7797): checking classloading for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:37,084 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 325e5f934bae06dfb2390407a9e9726f, server=843d83d51ba0,38659,1732104813354 because future has completed 2024-11-20T12:13:37,084 INFO [StoreOpener-5f89ec3b6d6dbb62bc1c5a7ca523c08c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:37,085 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1093): writing seq id for fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:37,087 INFO [StoreOpener-5f89ec3b6d6dbb62bc1c5a7ca523c08c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f89ec3b6d6dbb62bc1c5a7ca523c08c columnFamilyName family 2024-11-20T12:13:37,087 DEBUG [StoreOpener-5f89ec3b6d6dbb62bc1c5a7ca523c08c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,088 INFO [StoreOpener-5f89ec3b6d6dbb62bc1c5a7ca523c08c-1 {}] regionserver.HStore(327): Store=5f89ec3b6d6dbb62bc1c5a7ca523c08c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,088 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:37,089 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 633a6ec91d322a9170f4899c41489180, server=843d83d51ba0,44663,1732104813203 because future has completed 2024-11-20T12:13:37,090 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:37,090 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:37,091 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:37,092 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:37,092 INFO [StoreOpener-03e3b3cc4355d1bb26b41172ffed46ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:37,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=10 2024-11-20T12:13:37,095 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 325e5f934bae06dfb2390407a9e9726f, server=843d83d51ba0,38659,1732104813354 in 236 msec 2024-11-20T12:13:37,096 INFO [StoreOpener-03e3b3cc4355d1bb26b41172ffed46ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03e3b3cc4355d1bb26b41172ffed46ca columnFamilyName family 2024-11-20T12:13:37,096 DEBUG [StoreOpener-03e3b3cc4355d1bb26b41172ffed46ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,097 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/fef27752c5c42f2f9602a38de6f56349/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,098 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:37,098 INFO [StoreOpener-03e3b3cc4355d1bb26b41172ffed46ca-1 {}] regionserver.HStore(327): Store=03e3b3cc4355d1bb26b41172ffed46ca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,098 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1114): Opened fef27752c5c42f2f9602a38de6f56349; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67729638, jitterRate=0.009250253438949585}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,098 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:37,099 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegion(1006): Region open journal for fef27752c5c42f2f9602a38de6f56349: Running coprocessor pre-open hook at 1732104817067Writing region info on filesystem at 1732104817067Initializing all the Stores at 1732104817069 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817069Cleaning up temporary data from old regions at 1732104817082 (+13 ms)Running coprocessor post-open hooks at 1732104817098 (+16 ms)Region opened successfully at 1732104817098 2024-11-20T12:13:37,100 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1038): replaying wal for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:37,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=325e5f934bae06dfb2390407a9e9726f, ASSIGN in 505 msec 2024-11-20T12:13:37,101 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2236): Post open deploy tasks for test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349., pid=43, masterSystemTime=1732104816977 2024-11-20T12:13:37,101 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:37,103 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:37,104 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1048): stopping wal replay for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:37,104 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1060): Cleaning up temporary data for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:37,105 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/5f89ec3b6d6dbb62bc1c5a7ca523c08c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,106 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] regionserver.HRegionServer(2266): Finished post open deploy task for test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:37,106 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 5f89ec3b6d6dbb62bc1c5a7ca523c08c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64439782, jitterRate=-0.03977242112159729}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,106 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=43}] handler.AssignRegionHandler(153): Opened test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:37,106 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:37,106 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:37,106 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 5f89ec3b6d6dbb62bc1c5a7ca523c08c: Running coprocessor pre-open hook at 1732104817080Writing region info on filesystem at 1732104817080Initializing all the Stores at 1732104817082 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817082Cleaning up temporary data from old regions at 1732104817092 (+10 ms)Running coprocessor post-open hooks at 1732104817106 (+14 ms)Region opened successfully at 1732104817106 2024-11-20T12:13:37,107 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 0a3d655f8e46d7b0303a071be25a13da, NAME => 'test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da.', STARTKEY => 'nnn', ENDKEY => 'ooo'} 2024-11-20T12:13:37,107 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:37,108 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,108 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c., pid=36, masterSystemTime=1732104816923 2024-11-20T12:13:37,108 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:37,108 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:37,111 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:37,112 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:37,112 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1093): writing seq id for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:37,112 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(132): Open test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:37,112 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7752): Opening region: {ENCODED => 6171c0ffde5383e6eb4635a4c76e3ad1, NAME => 'test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1.', STARTKEY => 'hhh', ENDKEY => 'iii'} 2024-11-20T12:13:37,113 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:37,113 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(898): Instantiated test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,113 INFO [StoreOpener-0a3d655f8e46d7b0303a071be25a13da-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:37,113 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7794): checking encryption for 6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:37,113 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(7797): checking classloading for 6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:37,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=16 2024-11-20T12:13:37,114 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 633a6ec91d322a9170f4899c41489180, server=843d83d51ba0,44663,1732104813203 in 318 msec 2024-11-20T12:13:37,117 INFO [StoreOpener-6171c0ffde5383e6eb4635a4c76e3ad1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:37,118 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/03e3b3cc4355d1bb26b41172ffed46ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,119 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1114): Opened 03e3b3cc4355d1bb26b41172ffed46ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63639991, jitterRate=-0.0516902357339859}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,119 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:37,119 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegion(1006): Region open journal for 03e3b3cc4355d1bb26b41172ffed46ca: Running coprocessor pre-open hook at 1732104817084Writing region info on filesystem at 1732104817084Initializing all the Stores at 1732104817086 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817086Cleaning up temporary data from old regions at 1732104817104 (+18 ms)Running coprocessor post-open hooks at 1732104817119 (+15 ms)Region opened successfully at 1732104817119 2024-11-20T12:13:37,121 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca., pid=52, masterSystemTime=1732104817075 2024-11-20T12:13:37,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=633a6ec91d322a9170f4899c41489180, ASSIGN in 524 msec 2024-11-20T12:13:37,121 INFO [StoreOpener-6171c0ffde5383e6eb4635a4c76e3ad1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6171c0ffde5383e6eb4635a4c76e3ad1 columnFamilyName family 2024-11-20T12:13:37,122 DEBUG [StoreOpener-6171c0ffde5383e6eb4635a4c76e3ad1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,123 INFO [StoreOpener-6171c0ffde5383e6eb4635a4c76e3ad1-1 {}] regionserver.HStore(327): Store=6171c0ffde5383e6eb4635a4c76e3ad1/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,124 INFO [StoreOpener-0a3d655f8e46d7b0303a071be25a13da-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0a3d655f8e46d7b0303a071be25a13da columnFamilyName family 2024-11-20T12:13:37,124 DEBUG [StoreOpener-0a3d655f8e46d7b0303a071be25a13da-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,125 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:37,125 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=52}] handler.AssignRegionHandler(153): Opened test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:37,125 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1038): replaying wal for 6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:37,125 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(132): Open test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:37,125 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7752): Opening region: {ENCODED => 99f9c663ffb10623454992ca39cafe6e, NAME => 'test,,1732104815671.99f9c663ffb10623454992ca39cafe6e.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-20T12:13:37,126 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:37,126 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(898): Instantiated test,,1732104815671.99f9c663ffb10623454992ca39cafe6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,126 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:37,126 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7794): checking encryption for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:37,126 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(7797): checking classloading for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:37,127 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:37,127 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:37,128 INFO [StoreOpener-0a3d655f8e46d7b0303a071be25a13da-1 {}] regionserver.HStore(327): Store=0a3d655f8e46d7b0303a071be25a13da/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,128 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:37,128 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1048): stopping wal replay for 6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:37,128 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1060): Cleaning up temporary data for 6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:37,129 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:37,131 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:37,131 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:37,131 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:37,131 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:37,132 INFO [StoreOpener-99f9c663ffb10623454992ca39cafe6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:37,134 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1093): writing seq id for 6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:37,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5f89ec3b6d6dbb62bc1c5a7ca523c08c, server=843d83d51ba0,44663,1732104813203 because future has completed 2024-11-20T12:13:37,136 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:37,139 INFO [StoreOpener-99f9c663ffb10623454992ca39cafe6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99f9c663ffb10623454992ca39cafe6e columnFamilyName family 2024-11-20T12:13:37,139 DEBUG [StoreOpener-99f9c663ffb10623454992ca39cafe6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,141 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,35591,1732104813307 because future has completed 2024-11-20T12:13:37,142 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=843d83d51ba0,44663,1732104813203, table=test, region=5f89ec3b6d6dbb62bc1c5a7ca523c08c. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T12:13:37,144 INFO [StoreOpener-99f9c663ffb10623454992ca39cafe6e-1 {}] regionserver.HStore(327): Store=99f9c663ffb10623454992ca39cafe6e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,144 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1038): replaying wal for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:37,145 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/0a3d655f8e46d7b0303a071be25a13da/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,145 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/6171c0ffde5383e6eb4635a4c76e3ad1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,145 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:37,146 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 0a3d655f8e46d7b0303a071be25a13da; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74923062, jitterRate=0.11644062399864197}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,146 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:37,146 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:37,146 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 0a3d655f8e46d7b0303a071be25a13da: Running coprocessor pre-open hook at 1732104817109Writing region info on filesystem at 1732104817109Initializing all the Stores at 1732104817111 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817111Cleaning up temporary data from old regions at 1732104817131 (+20 ms)Running coprocessor post-open hooks at 1732104817146 (+15 ms)Region opened successfully at 1732104817146 2024-11-20T12:13:37,147 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1114): Opened 6171c0ffde5383e6eb4635a4c76e3ad1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61066013, jitterRate=-0.0900454968214035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,148 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1048): stopping wal replay for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:37,148 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1060): Cleaning up temporary data for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:37,148 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da., pid=39, masterSystemTime=1732104816977 2024-11-20T12:13:37,148 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:37,148 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegion(1006): Region open journal for 6171c0ffde5383e6eb4635a4c76e3ad1: Running coprocessor pre-open hook at 1732104817113Writing region info on filesystem at 1732104817113Initializing all the Stores at 1732104817116 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817116Cleaning up temporary data from old regions at 1732104817128 (+12 ms)Running coprocessor post-open hooks at 1732104817148 (+20 ms)Region opened successfully at 1732104817148 2024-11-20T12:13:37,151 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2236): Post open deploy tasks for test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1., pid=37, masterSystemTime=1732104816923 2024-11-20T12:13:37,152 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:37,152 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:37,153 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(132): Open test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:37,153 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1093): writing seq id for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:37,153 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7752): Opening region: {ENCODED => 312c8b2fa94f26f46326bffa7ae2bcff, NAME => 'test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff.', STARTKEY => 'ttt', ENDKEY => 'uuu'} 2024-11-20T12:13:37,153 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:37,153 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(898): Instantiated test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,153 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7794): checking encryption for 312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:37,154 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(7797): checking classloading for 312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:37,157 INFO [StoreOpener-312c8b2fa94f26f46326bffa7ae2bcff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:37,159 INFO [StoreOpener-312c8b2fa94f26f46326bffa7ae2bcff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 312c8b2fa94f26f46326bffa7ae2bcff columnFamilyName family 2024-11-20T12:13:37,159 DEBUG [StoreOpener-312c8b2fa94f26f46326bffa7ae2bcff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,160 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=843d83d51ba0,35591,1732104813307, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T12:13:37,160 INFO [StoreOpener-312c8b2fa94f26f46326bffa7ae2bcff-1 {}] regionserver.HStore(327): Store=312c8b2fa94f26f46326bffa7ae2bcff/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,161 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1038): replaying wal for 312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:37,161 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] regionserver.HRegionServer(2266): Finished post open deploy task for test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:37,161 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=37}] handler.AssignRegionHandler(153): Opened test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:37,161 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:37,161 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 366d4b256c89d7860c4d7747ae8d7cc6, NAME => 'test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6.', STARTKEY => 'zzz', ENDKEY => ''} 2024-11-20T12:13:37,162 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:37,162 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,162 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:37,162 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:37,162 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:37,163 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:37,164 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1048): stopping wal replay for 312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:37,165 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1060): Cleaning up temporary data for 312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:37,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=27 2024-11-20T12:13:37,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 5f89ec3b6d6dbb62bc1c5a7ca523c08c, server=843d83d51ba0,44663,1732104813203 in 352 msec 2024-11-20T12:13:37,168 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1093): writing seq id for 312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:37,168 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/99f9c663ffb10623454992ca39cafe6e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,169 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1114): Opened 99f9c663ffb10623454992ca39cafe6e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74384179, jitterRate=0.10841064155101776}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,170 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:37,170 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegion(1006): Region open journal for 99f9c663ffb10623454992ca39cafe6e: Running coprocessor pre-open hook at 1732104817126Writing region info on filesystem at 1732104817126Initializing all the Stores at 1732104817128 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817128Cleaning up temporary data from old regions at 1732104817148 (+20 ms)Running coprocessor post-open hooks at 1732104817170 (+22 ms)Region opened successfully at 1732104817170 2024-11-20T12:13:37,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, ASSIGN in 578 msec 2024-11-20T12:13:37,172 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732104815671.99f9c663ffb10623454992ca39cafe6e., pid=56, masterSystemTime=1732104817075 2024-11-20T12:13:37,176 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=12 updating hbase:meta row=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:37,174 INFO [StoreOpener-366d4b256c89d7860c4d7747ae8d7cc6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:37,178 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:37,178 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=56}] handler.AssignRegionHandler(153): Opened test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:37,178 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(132): Open test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:37,178 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7752): Opening region: {ENCODED => 2186bb28c010fe9bd50ae06c405d542b, NAME => 'test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-20T12:13:37,179 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:37,179 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(898): Instantiated test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,179 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7794): checking encryption for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:37,179 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(7797): checking classloading for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:37,181 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=99f9c663ffb10623454992ca39cafe6e, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:37,181 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=12, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6171c0ffde5383e6eb4635a4c76e3ad1, server=843d83d51ba0,44663,1732104813203 because future has completed 2024-11-20T12:13:37,185 INFO [StoreOpener-366d4b256c89d7860c4d7747ae8d7cc6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 366d4b256c89d7860c4d7747ae8d7cc6 columnFamilyName family 2024-11-20T12:13:37,185 DEBUG [StoreOpener-366d4b256c89d7860c4d7747ae8d7cc6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,185 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=56, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,35591,1732104813307 because future has completed 2024-11-20T12:13:37,186 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/312c8b2fa94f26f46326bffa7ae2bcff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,187 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1114): Opened 312c8b2fa94f26f46326bffa7ae2bcff; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73659798, jitterRate=0.0976165235042572}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,187 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:37,188 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegion(1006): Region open journal for 312c8b2fa94f26f46326bffa7ae2bcff: Running coprocessor pre-open hook at 1732104817154Writing region info on filesystem at 1732104817154Initializing all the Stores at 1732104817156 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817156Cleaning up temporary data from old regions at 1732104817165 (+9 ms)Running coprocessor post-open hooks at 1732104817187 (+22 ms)Region opened successfully at 1732104817187 2024-11-20T12:13:37,188 INFO [StoreOpener-366d4b256c89d7860c4d7747ae8d7cc6-1 {}] regionserver.HStore(327): Store=366d4b256c89d7860c4d7747ae8d7cc6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,188 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:37,190 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff., pid=44, masterSystemTime=1732104816977 2024-11-20T12:13:37,190 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:37,191 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:37,193 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:37,193 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:37,196 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:37,196 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=44}] handler.AssignRegionHandler(153): Opened test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:37,196 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(132): Open test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:37,196 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7752): Opening region: {ENCODED => dec6f7c7f05d870bae6d81029b3676e2, NAME => 'test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2.', STARTKEY => 'iii', ENDKEY => 'jjj'} 2024-11-20T12:13:37,197 INFO [StoreOpener-2186bb28c010fe9bd50ae06c405d542b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:37,197 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:37,197 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:37,197 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(898): Instantiated test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,198 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7794): checking encryption for dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:37,198 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(7797): checking classloading for dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:37,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=12 2024-11-20T12:13:37,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=12, state=SUCCESS, hasLock=false; OpenRegionProcedure 6171c0ffde5383e6eb4635a4c76e3ad1, server=843d83d51ba0,44663,1732104813203 in 382 msec 2024-11-20T12:13:37,205 INFO [StoreOpener-2186bb28c010fe9bd50ae06c405d542b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2186bb28c010fe9bd50ae06c405d542b columnFamilyName family 2024-11-20T12:13:37,206 DEBUG [StoreOpener-2186bb28c010fe9bd50ae06c405d542b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,207 INFO [StoreOpener-2186bb28c010fe9bd50ae06c405d542b-1 {}] regionserver.HStore(327): Store=2186bb28c010fe9bd50ae06c405d542b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,207 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1038): replaying wal for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:37,208 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:37,209 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:37,209 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=6171c0ffde5383e6eb4635a4c76e3ad1, ASSIGN in 615 msec 2024-11-20T12:13:37,210 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1048): stopping wal replay for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:37,210 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1060): Cleaning up temporary data for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:37,212 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=24 updating hbase:meta row=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:37,215 INFO [StoreOpener-dec6f7c7f05d870bae6d81029b3676e2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:37,215 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1093): writing seq id for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:37,216 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=44, ppid=24, state=RUNNABLE, hasLock=false; OpenRegionProcedure 312c8b2fa94f26f46326bffa7ae2bcff, server=843d83d51ba0,38659,1732104813354 because future has completed 2024-11-20T12:13:37,218 INFO [StoreOpener-dec6f7c7f05d870bae6d81029b3676e2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dec6f7c7f05d870bae6d81029b3676e2 columnFamilyName family 2024-11-20T12:13:37,219 DEBUG [StoreOpener-dec6f7c7f05d870bae6d81029b3676e2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,220 INFO [StoreOpener-dec6f7c7f05d870bae6d81029b3676e2-1 {}] regionserver.HStore(327): Store=dec6f7c7f05d870bae6d81029b3676e2/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,220 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1038): replaying wal for dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:37,221 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:37,221 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/366d4b256c89d7860c4d7747ae8d7cc6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,222 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:37,222 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 366d4b256c89d7860c4d7747ae8d7cc6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69831767, jitterRate=0.040574416518211365}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,222 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:37,223 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 366d4b256c89d7860c4d7747ae8d7cc6: Running coprocessor pre-open hook at 1732104817162Writing region info on filesystem at 1732104817162Initializing all the Stores at 1732104817165 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817165Cleaning up temporary data from old regions at 1732104817194 (+29 ms)Running coprocessor post-open hooks at 1732104817222 (+28 ms)Region opened successfully at 1732104817222 2024-11-20T12:13:37,223 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1048): stopping wal replay for dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:37,223 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1060): Cleaning up temporary data for dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:37,224 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6., pid=31, masterSystemTime=1732104816923 2024-11-20T12:13:37,224 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/2186bb28c010fe9bd50ae06c405d542b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,225 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1114): Opened 2186bb28c010fe9bd50ae06c405d542b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60716809, jitterRate=-0.09524904191493988}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,225 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:37,225 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegion(1006): Region open journal for 2186bb28c010fe9bd50ae06c405d542b: Running coprocessor pre-open hook at 1732104817179Writing region info on filesystem at 1732104817179Initializing all the Stores at 1732104817181 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817181Cleaning up temporary data from old regions at 1732104817210 (+29 ms)Running coprocessor post-open hooks at 1732104817225 (+15 ms)Region opened successfully at 1732104817225 2024-11-20T12:13:37,227 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1093): writing seq id for dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:37,230 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/dec6f7c7f05d870bae6d81029b3676e2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,231 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b., pid=48, masterSystemTime=1732104817075 2024-11-20T12:13:37,232 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1114): Opened dec6f7c7f05d870bae6d81029b3676e2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69642754, jitterRate=0.03775790333747864}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,232 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:37,232 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegion(1006): Region open journal for dec6f7c7f05d870bae6d81029b3676e2: Running coprocessor pre-open hook at 1732104817198Writing region info on filesystem at 1732104817198Initializing all the Stores at 1732104817202 (+4 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817202Cleaning up temporary data from old regions at 1732104817223 (+21 ms)Running coprocessor post-open hooks at 1732104817232 (+9 ms)Region opened successfully at 1732104817232 2024-11-20T12:13:37,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=24 2024-11-20T12:13:37,233 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=24, state=SUCCESS, hasLock=false; OpenRegionProcedure 312c8b2fa94f26f46326bffa7ae2bcff, server=843d83d51ba0,38659,1732104813354 in 340 msec 2024-11-20T12:13:37,233 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2236): Post open deploy tasks for test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2., pid=42, masterSystemTime=1732104816977 2024-11-20T12:13:37,235 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:37,235 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:37,235 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(132): Open test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:37,235 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7752): Opening region: {ENCODED => ae055eeceea830ff461b6b4dc158461e, NAME => 'test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e.', STARTKEY => 'ccc', ENDKEY => 'ddd'} 2024-11-20T12:13:37,236 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:37,236 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(898): Instantiated test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,236 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7794): checking encryption for ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:37,236 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(7797): checking classloading for ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:37,238 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:37,238 INFO [StoreOpener-ae055eeceea830ff461b6b4dc158461e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:37,238 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=48}] handler.AssignRegionHandler(153): Opened test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:37,239 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(132): Open test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:37,239 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7752): Opening region: {ENCODED => abac5b1bb17e965219c5ca01a8ebc325, NAME => 'test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-20T12:13:37,239 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=30 updating hbase:meta row=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:37,239 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:37,239 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(898): Instantiated test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,239 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7794): checking encryption for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:37,239 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(7797): checking classloading for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:37,243 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] regionserver.HRegionServer(2266): Finished post open deploy task for test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:37,243 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=42}] handler.AssignRegionHandler(153): Opened test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:37,244 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(132): Open test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:37,244 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7752): Opening region: {ENCODED => 8978fd254a3dd3cd4570b36a1c18b8cb, NAME => 'test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb.', STARTKEY => 'yyy', ENDKEY => 'zzz'} 2024-11-20T12:13:37,244 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=31, ppid=30, state=RUNNABLE, hasLock=true; OpenRegionProcedure 366d4b256c89d7860c4d7747ae8d7cc6, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:37,245 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:37,245 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(898): Instantiated test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,246 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7794): checking encryption for 8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:37,246 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(7797): checking classloading for 8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:37,248 INFO [StoreOpener-ae055eeceea830ff461b6b4dc158461e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ae055eeceea830ff461b6b4dc158461e columnFamilyName family 2024-11-20T12:13:37,249 DEBUG [StoreOpener-ae055eeceea830ff461b6b4dc158461e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,250 INFO [StoreOpener-ae055eeceea830ff461b6b4dc158461e-1 {}] regionserver.HStore(327): Store=ae055eeceea830ff461b6b4dc158461e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,250 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1038): replaying wal for ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:37,252 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:37,253 INFO [StoreOpener-abac5b1bb17e965219c5ca01a8ebc325-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:37,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-20T12:13:37,254 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; OpenRegionProcedure 366d4b256c89d7860c4d7747ae8d7cc6, server=843d83d51ba0,44663,1732104813203 in 477 msec 2024-11-20T12:13:37,254 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:37,255 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1048): stopping wal replay for ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:37,255 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1060): Cleaning up temporary data for ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:37,256 INFO [StoreOpener-abac5b1bb17e965219c5ca01a8ebc325-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abac5b1bb17e965219c5ca01a8ebc325 columnFamilyName family 2024-11-20T12:13:37,257 DEBUG [StoreOpener-abac5b1bb17e965219c5ca01a8ebc325-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,258 INFO [StoreOpener-abac5b1bb17e965219c5ca01a8ebc325-1 {}] regionserver.HStore(327): Store=abac5b1bb17e965219c5ca01a8ebc325/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,258 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1038): replaying wal for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:37,259 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:37,259 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1093): writing seq id for ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:37,259 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:37,260 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1048): stopping wal replay for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:37,260 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1060): Cleaning up temporary data for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:37,262 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=366d4b256c89d7860c4d7747ae8d7cc6, ASSIGN in 664 msec 2024-11-20T12:13:37,262 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1093): writing seq id for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:37,264 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=843d83d51ba0,35591,1732104813307, table=test, region=99f9c663ffb10623454992ca39cafe6e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T12:13:37,264 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=843d83d51ba0,35591,1732104813307, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T12:13:37,265 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=13 updating hbase:meta row=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:37,267 INFO [StoreOpener-8978fd254a3dd3cd4570b36a1c18b8cb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:37,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=42, ppid=13, state=RUNNABLE, hasLock=false; OpenRegionProcedure dec6f7c7f05d870bae6d81029b3676e2, server=843d83d51ba0,38659,1732104813354 because future has completed 2024-11-20T12:13:37,270 INFO [StoreOpener-8978fd254a3dd3cd4570b36a1c18b8cb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8978fd254a3dd3cd4570b36a1c18b8cb columnFamilyName family 2024-11-20T12:13:37,270 DEBUG [StoreOpener-8978fd254a3dd3cd4570b36a1c18b8cb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,271 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/abac5b1bb17e965219c5ca01a8ebc325/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,271 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/ae055eeceea830ff461b6b4dc158461e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,271 INFO [StoreOpener-8978fd254a3dd3cd4570b36a1c18b8cb-1 {}] regionserver.HStore(327): Store=8978fd254a3dd3cd4570b36a1c18b8cb/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,271 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1038): replaying wal for 8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:37,271 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=28 updating hbase:meta row=2186bb28c010fe9bd50ae06c405d542b, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:37,272 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1114): Opened abac5b1bb17e965219c5ca01a8ebc325; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73371821, jitterRate=0.09332533180713654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,272 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:37,272 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegion(1006): Region open journal for abac5b1bb17e965219c5ca01a8ebc325: Running coprocessor pre-open hook at 1732104817240Writing region info on filesystem at 1732104817240Initializing all the Stores at 1732104817241 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817241Cleaning up temporary data from old regions at 1732104817260 (+19 ms)Running coprocessor post-open hooks at 1732104817272 (+12 ms)Region opened successfully at 1732104817272 2024-11-20T12:13:37,273 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:37,273 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:37,273 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1114): Opened ae055eeceea830ff461b6b4dc158461e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63428554, jitterRate=-0.05484089255332947}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,274 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1048): stopping wal replay for 8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:37,274 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:37,274 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1060): Cleaning up temporary data for 8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:37,274 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegion(1006): Region open journal for ae055eeceea830ff461b6b4dc158461e: Running coprocessor pre-open hook at 1732104817236Writing region info on filesystem at 1732104817236Initializing all the Stores at 1732104817238 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817238Cleaning up temporary data from old regions at 1732104817255 (+17 ms)Running coprocessor post-open hooks at 1732104817274 (+19 ms)Region opened successfully at 1732104817274 2024-11-20T12:13:37,277 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325., pid=50, masterSystemTime=1732104817075 2024-11-20T12:13:37,277 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e., pid=46, masterSystemTime=1732104816923 2024-11-20T12:13:37,278 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=28, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,35591,1732104813307 because future has completed 2024-11-20T12:13:37,279 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1093): writing seq id for 8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:37,281 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:37,281 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=50}] handler.AssignRegionHandler(153): Opened test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:37,281 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(132): Open test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:37,281 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7752): Opening region: {ENCODED => f42c0bd7eed4146b0b4502022341fd1d, NAME => 'test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-20T12:13:37,282 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:37,282 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(898): Instantiated test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,282 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7794): checking encryption for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:37,282 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(7797): checking classloading for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:37,284 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:37,284 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=46}] handler.AssignRegionHandler(153): Opened test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:37,285 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(132): Open test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:37,285 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b2ba9c7430b8a524b3966df315ad45c, NAME => 'test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c.', STARTKEY => 'ddd', ENDKEY => 'eee'} 2024-11-20T12:13:37,286 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:37,286 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(898): Instantiated test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,286 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7794): checking encryption for 9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:37,286 INFO [StoreOpener-f42c0bd7eed4146b0b4502022341fd1d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:37,286 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(7797): checking classloading for 9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:37,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=42, resume processing ppid=13 2024-11-20T12:13:37,287 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8978fd254a3dd3cd4570b36a1c18b8cb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, ppid=13, state=SUCCESS, hasLock=false; OpenRegionProcedure dec6f7c7f05d870bae6d81029b3676e2, server=843d83d51ba0,38659,1732104813354 in 419 msec 2024-11-20T12:13:37,289 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1114): Opened 8978fd254a3dd3cd4570b36a1c18b8cb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72714410, jitterRate=0.08352914452552795}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,289 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:37,289 INFO [StoreOpener-f42c0bd7eed4146b0b4502022341fd1d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f42c0bd7eed4146b0b4502022341fd1d columnFamilyName family 2024-11-20T12:13:37,289 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegion(1006): Region open journal for 8978fd254a3dd3cd4570b36a1c18b8cb: Running coprocessor pre-open hook at 1732104817246Writing region info on filesystem at 1732104817246Initializing all the Stores at 1732104817248 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817248Cleaning up temporary data from old regions at 1732104817274 (+26 ms)Running coprocessor post-open hooks at 1732104817289 (+15 ms)Region opened successfully at 1732104817289 2024-11-20T12:13:37,289 DEBUG [StoreOpener-f42c0bd7eed4146b0b4502022341fd1d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,290 INFO [StoreOpener-f42c0bd7eed4146b0b4502022341fd1d-1 {}] regionserver.HStore(327): Store=f42c0bd7eed4146b0b4502022341fd1d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,290 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2236): Post open deploy tasks for test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb., pid=38, masterSystemTime=1732104816977 2024-11-20T12:13:37,291 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1038): replaying wal for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:37,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=dec6f7c7f05d870bae6d81029b3676e2, ASSIGN in 698 msec 2024-11-20T12:13:37,293 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:37,293 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:37,293 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:37,294 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] regionserver.HRegionServer(2266): Finished post open deploy task for test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:37,294 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=38}] handler.AssignRegionHandler(153): Opened test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:37,294 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(132): Open test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:37,294 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1048): stopping wal replay for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:37,294 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1060): Cleaning up temporary data for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:37,294 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7752): Opening region: {ENCODED => 23c5a5a22a2d09813d1ff472bb76f407, NAME => 'test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407.', STARTKEY => 'bbb', ENDKEY => 'ccc'} 2024-11-20T12:13:37,295 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:37,295 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(898): Instantiated test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,295 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7794): checking encryption for 23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:37,295 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(7797): checking classloading for 23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:37,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=7, state=RUNNABLE, hasLock=true; OpenRegionProcedure ae055eeceea830ff461b6b4dc158461e, server=843d83d51ba0,44663,1732104813203 because future has completed 2024-11-20T12:13:37,303 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1093): writing seq id for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:37,308 INFO [StoreOpener-9b2ba9c7430b8a524b3966df315ad45c-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:37,309 INFO [StoreOpener-23c5a5a22a2d09813d1ff472bb76f407-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:37,311 INFO [StoreOpener-9b2ba9c7430b8a524b3966df315ad45c-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b2ba9c7430b8a524b3966df315ad45c columnFamilyName family 2024-11-20T12:13:37,311 DEBUG [StoreOpener-9b2ba9c7430b8a524b3966df315ad45c-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,311 INFO [StoreOpener-23c5a5a22a2d09813d1ff472bb76f407-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23c5a5a22a2d09813d1ff472bb76f407 columnFamilyName family 2024-11-20T12:13:37,311 DEBUG [StoreOpener-23c5a5a22a2d09813d1ff472bb76f407-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,312 INFO [StoreOpener-9b2ba9c7430b8a524b3966df315ad45c-1 {}] regionserver.HStore(327): Store=9b2ba9c7430b8a524b3966df315ad45c/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,313 INFO [StoreOpener-23c5a5a22a2d09813d1ff472bb76f407-1 {}] regionserver.HStore(327): Store=23c5a5a22a2d09813d1ff472bb76f407/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,313 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/f42c0bd7eed4146b0b4502022341fd1d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,315 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1038): replaying wal for 9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:37,318 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1038): replaying wal for 23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:37,318 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1114): Opened f42c0bd7eed4146b0b4502022341fd1d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59121176, jitterRate=-0.1190258264541626}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,319 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:37,319 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:37,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=7 2024-11-20T12:13:37,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure ae055eeceea830ff461b6b4dc158461e, server=843d83d51ba0,44663,1732104813203 in 411 msec 2024-11-20T12:13:37,319 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegion(1006): Region open journal for f42c0bd7eed4146b0b4502022341fd1d: Running coprocessor pre-open hook at 1732104817282Writing region info on filesystem at 1732104817282Initializing all the Stores at 1732104817284 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817284Cleaning up temporary data from old regions at 1732104817294 (+10 ms)Running coprocessor post-open hooks at 1732104817319 (+25 ms)Region opened successfully at 1732104817319 2024-11-20T12:13:37,319 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:37,320 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1048): stopping wal replay for 9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:37,321 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1060): Cleaning up temporary data for 9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:37,321 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d., pid=49, masterSystemTime=1732104817075 2024-11-20T12:13:37,324 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=ae055eeceea830ff461b6b4dc158461e, ASSIGN in 729 msec 2024-11-20T12:13:37,325 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1093): writing seq id for 9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:37,327 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:37,327 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=49}] handler.AssignRegionHandler(153): Opened test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:37,327 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:37,327 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(132): Open test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:37,328 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7752): Opening region: {ENCODED => 8448ac25ea038ac8c63c9da8906d8673, NAME => 'test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-20T12:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(898): Instantiated test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7794): checking encryption for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:37,330 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(7797): checking classloading for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:37,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8978fd254a3dd3cd4570b36a1c18b8cb, server=843d83d51ba0,38659,1732104813354 because future has completed 2024-11-20T12:13:37,336 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=17 updating hbase:meta row=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:37,338 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:37,341 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1048): stopping wal replay for 23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:37,341 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1060): Cleaning up temporary data for 23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:37,342 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=17, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,35591,1732104813307 because future has completed 2024-11-20T12:13:37,344 INFO [StoreOpener-8448ac25ea038ac8c63c9da8906d8673-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:37,344 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/9b2ba9c7430b8a524b3966df315ad45c/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,345 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1093): writing seq id for 23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:37,345 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1114): Opened 9b2ba9c7430b8a524b3966df315ad45c; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71624062, jitterRate=0.06728169322013855}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,345 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:37,346 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegion(1006): Region open journal for 9b2ba9c7430b8a524b3966df315ad45c: Running coprocessor pre-open hook at 1732104817286Writing region info on filesystem at 1732104817286Initializing all the Stores at 1732104817288 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817288Cleaning up temporary data from old regions at 1732104817321 (+33 ms)Running coprocessor post-open hooks at 1732104817345 (+24 ms)Region opened successfully at 1732104817345 2024-11-20T12:13:37,347 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c., pid=47, masterSystemTime=1732104816923 2024-11-20T12:13:37,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=29 2024-11-20T12:13:37,347 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 8978fd254a3dd3cd4570b36a1c18b8cb, server=843d83d51ba0,38659,1732104813354 in 524 msec 2024-11-20T12:13:37,351 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:37,351 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=47}] handler.AssignRegionHandler(153): Opened test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:37,352 INFO [StoreOpener-8448ac25ea038ac8c63c9da8906d8673-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8448ac25ea038ac8c63c9da8906d8673 columnFamilyName family 2024-11-20T12:13:37,352 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8978fd254a3dd3cd4570b36a1c18b8cb, ASSIGN in 757 msec 2024-11-20T12:13:37,352 DEBUG [StoreOpener-8448ac25ea038ac8c63c9da8906d8673-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,353 INFO [StoreOpener-8448ac25ea038ac8c63c9da8906d8673-1 {}] regionserver.HStore(327): Store=8448ac25ea038ac8c63c9da8906d8673/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,354 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1038): replaying wal for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:37,354 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=8 updating hbase:meta row=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:37,357 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:37,358 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:37,360 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1048): stopping wal replay for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:37,361 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1060): Cleaning up temporary data for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:37,361 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/23c5a5a22a2d09813d1ff472bb76f407/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,362 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1114): Opened 23c5a5a22a2d09813d1ff472bb76f407; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72787389, jitterRate=0.08461661636829376}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,362 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=8, state=RUNNABLE, hasLock=false; OpenRegionProcedure 9b2ba9c7430b8a524b3966df315ad45c, server=843d83d51ba0,44663,1732104813203 because future has completed 2024-11-20T12:13:37,362 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:37,362 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegion(1006): Region open journal for 23c5a5a22a2d09813d1ff472bb76f407: Running coprocessor pre-open hook at 1732104817295Writing region info on filesystem at 1732104817295Initializing all the Stores at 1732104817302 (+7 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817302Cleaning up temporary data from old regions at 1732104817341 (+39 ms)Running coprocessor post-open hooks at 1732104817362 (+21 ms)Region opened successfully at 1732104817362 2024-11-20T12:13:37,364 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2236): Post open deploy tasks for test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407., pid=40, masterSystemTime=1732104816977 2024-11-20T12:13:37,364 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1093): writing seq id for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:37,371 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8448ac25ea038ac8c63c9da8906d8673/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,371 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1114): Opened 8448ac25ea038ac8c63c9da8906d8673; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73214839, jitterRate=0.09098611772060394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,372 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:37,372 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegion(1006): Region open journal for 8448ac25ea038ac8c63c9da8906d8673: Running coprocessor pre-open hook at 1732104817331Writing region info on filesystem at 1732104817331Initializing all the Stores at 1732104817333 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817333Cleaning up temporary data from old regions at 1732104817361 (+28 ms)Running coprocessor post-open hooks at 1732104817372 (+11 ms)Region opened successfully at 1732104817372 2024-11-20T12:13:37,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=17 2024-11-20T12:13:37,373 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=17, state=SUCCESS, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,35591,1732104813307 in 435 msec 2024-11-20T12:13:37,373 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=843d83d51ba0,35591,1732104813307, table=test, region=99f9c663ffb10623454992ca39cafe6e. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T12:13:37,373 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=843d83d51ba0,35591,1732104813307, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T12:13:37,373 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=843d83d51ba0,35591,1732104813307, table=test, region=2186bb28c010fe9bd50ae06c405d542b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T12:13:37,374 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] regionserver.HRegionServer(2266): Finished post open deploy task for test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:37,374 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=40}] handler.AssignRegionHandler(153): Opened test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:37,374 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673., pid=55, masterSystemTime=1732104817075 2024-11-20T12:13:37,377 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN in 783 msec 2024-11-20T12:13:37,378 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:37,379 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=55}] handler.AssignRegionHandler(153): Opened test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:37,379 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:37,379 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:37,379 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 45e15b0320018bda387cb884d175aab6, NAME => 'test,sss,1732104815671.45e15b0320018bda387cb884d175aab6.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-20T12:13:37,379 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:37,379 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated test,sss,1732104815671.45e15b0320018bda387cb884d175aab6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,380 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:37,380 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:37,382 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=9 updating hbase:meta row=8448ac25ea038ac8c63c9da8906d8673, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:37,382 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 23c5a5a22a2d09813d1ff472bb76f407, server=843d83d51ba0,38659,1732104813354 because future has completed 2024-11-20T12:13:37,383 INFO [StoreOpener-45e15b0320018bda387cb884d175aab6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:37,386 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=9, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8448ac25ea038ac8c63c9da8906d8673, server=843d83d51ba0,35591,1732104813307 because future has completed 2024-11-20T12:13:37,387 INFO [StoreOpener-45e15b0320018bda387cb884d175aab6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e15b0320018bda387cb884d175aab6 columnFamilyName family 2024-11-20T12:13:37,387 DEBUG [StoreOpener-45e15b0320018bda387cb884d175aab6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,389 INFO [StoreOpener-45e15b0320018bda387cb884d175aab6-1 {}] regionserver.HStore(327): Store=45e15b0320018bda387cb884d175aab6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,389 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:37,390 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:37,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=6 2024-11-20T12:13:37,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 23c5a5a22a2d09813d1ff472bb76f407, server=843d83d51ba0,38659,1732104813354 in 545 msec 2024-11-20T12:13:37,391 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:37,391 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:37,392 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:37,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=23c5a5a22a2d09813d1ff472bb76f407, ASSIGN in 801 msec 2024-11-20T12:13:37,395 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:37,398 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/45e15b0320018bda387cb884d175aab6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=9 2024-11-20T12:13:37,399 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 45e15b0320018bda387cb884d175aab6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69265855, jitterRate=0.03214167058467865}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,399 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:37,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=9, state=SUCCESS, hasLock=false; OpenRegionProcedure 8448ac25ea038ac8c63c9da8906d8673, server=843d83d51ba0,35591,1732104813307 in 395 msec 2024-11-20T12:13:37,400 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 45e15b0320018bda387cb884d175aab6: Running coprocessor pre-open hook at 1732104817380Writing region info on filesystem at 1732104817380Initializing all the Stores at 1732104817382 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817382Cleaning up temporary data from old regions at 1732104817392 (+10 ms)Running coprocessor post-open hooks at 1732104817400 (+8 ms)Region opened successfully at 1732104817400 2024-11-20T12:13:37,401 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732104815671.45e15b0320018bda387cb884d175aab6., pid=53, masterSystemTime=1732104817075 2024-11-20T12:13:37,402 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN in 810 msec 2024-11-20T12:13:37,404 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:37,404 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:37,404 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:37,404 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b4c46659583192ce01a652d7688d63d, NAME => 'test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-20T12:13:37,405 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:37,405 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,405 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:37,405 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:37,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=8 2024-11-20T12:13:37,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=8, state=SUCCESS, hasLock=false; OpenRegionProcedure 9b2ba9c7430b8a524b3966df315ad45c, server=843d83d51ba0,44663,1732104813203 in 498 msec 2024-11-20T12:13:37,407 INFO [StoreOpener-0b4c46659583192ce01a652d7688d63d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:37,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=9b2ba9c7430b8a524b3966df315ad45c, ASSIGN in 817 msec 2024-11-20T12:13:37,409 INFO [StoreOpener-0b4c46659583192ce01a652d7688d63d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b4c46659583192ce01a652d7688d63d columnFamilyName family 2024-11-20T12:13:37,410 DEBUG [StoreOpener-0b4c46659583192ce01a652d7688d63d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,410 INFO [StoreOpener-0b4c46659583192ce01a652d7688d63d-1 {}] regionserver.HStore(327): Store=0b4c46659583192ce01a652d7688d63d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,410 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=45e15b0320018bda387cb884d175aab6, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:37,411 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:37,411 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:37,412 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:37,413 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:37,413 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:37,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=23, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,35591,1732104813307 because future has completed 2024-11-20T12:13:37,415 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=11 updating hbase:meta row=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:37,416 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:37,418 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=11, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,35591,1732104813307 because future has completed 2024-11-20T12:13:37,421 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/0b4c46659583192ce01a652d7688d63d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,421 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 0b4c46659583192ce01a652d7688d63d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74169490, jitterRate=0.1052115261554718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,422 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:37,422 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 0b4c46659583192ce01a652d7688d63d: Running coprocessor pre-open hook at 1732104817405Writing region info on filesystem at 1732104817405Initializing all the Stores at 1732104817407 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817407Cleaning up temporary data from old regions at 1732104817413 (+6 ms)Running coprocessor post-open hooks at 1732104817422 (+9 ms)Region opened successfully at 1732104817422 2024-11-20T12:13:37,423 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d., pid=51, masterSystemTime=1732104817075 2024-11-20T12:13:37,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=23 2024-11-20T12:13:37,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=23, state=SUCCESS, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,35591,1732104813307 in 458 msec 2024-11-20T12:13:37,426 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:37,426 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:37,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN in 834 msec 2024-11-20T12:13:37,426 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(132): Open test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:37,427 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7752): Opening region: {ENCODED => 113951a1eeaf21f60b028a18a0a1a80b, NAME => 'test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-20T12:13:37,427 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:37,427 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(898): Instantiated test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:37,427 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7794): checking encryption for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:37,427 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(7797): checking classloading for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:37,427 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=14 updating hbase:meta row=0b4c46659583192ce01a652d7688d63d, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:37,430 INFO [StoreOpener-113951a1eeaf21f60b028a18a0a1a80b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:37,430 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=14, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b4c46659583192ce01a652d7688d63d, server=843d83d51ba0,35591,1732104813307 because future has completed 2024-11-20T12:13:37,432 INFO [StoreOpener-113951a1eeaf21f60b028a18a0a1a80b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 113951a1eeaf21f60b028a18a0a1a80b columnFamilyName family 2024-11-20T12:13:37,432 DEBUG [StoreOpener-113951a1eeaf21f60b028a18a0a1a80b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:37,433 INFO [StoreOpener-113951a1eeaf21f60b028a18a0a1a80b-1 {}] regionserver.HStore(327): Store=113951a1eeaf21f60b028a18a0a1a80b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:37,433 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1038): replaying wal for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:37,434 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:37,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=11 2024-11-20T12:13:37,435 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=11, state=SUCCESS, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,35591,1732104813307 in 494 msec 2024-11-20T12:13:37,435 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:37,435 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1048): stopping wal replay for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:37,435 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1060): Cleaning up temporary data for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:37,437 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1093): writing seq id for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:37,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN in 845 msec 2024-11-20T12:13:37,441 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/113951a1eeaf21f60b028a18a0a1a80b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-20T12:13:37,441 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1114): Opened 113951a1eeaf21f60b028a18a0a1a80b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74196052, jitterRate=0.10560733079910278}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:37,442 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:37,442 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegion(1006): Region open journal for 113951a1eeaf21f60b028a18a0a1a80b: Running coprocessor pre-open hook at 1732104817428Writing region info on filesystem at 1732104817428Initializing all the Stores at 1732104817429 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104817429Cleaning up temporary data from old regions at 1732104817435 (+6 ms)Running coprocessor post-open hooks at 1732104817442 (+7 ms)Region opened successfully at 1732104817442 2024-11-20T12:13:37,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=14 2024-11-20T12:13:37,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=14, state=SUCCESS, hasLock=false; OpenRegionProcedure 0b4c46659583192ce01a652d7688d63d, server=843d83d51ba0,35591,1732104813307 in 495 msec 2024-11-20T12:13:37,443 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b., pid=54, masterSystemTime=1732104817075 2024-11-20T12:13:37,444 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN in 852 msec 2024-11-20T12:13:37,446 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:37,446 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=54}] handler.AssignRegionHandler(153): Opened test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:37,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=28 2024-11-20T12:13:37,449 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=28, state=SUCCESS, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,35591,1732104813307 in 527 msec 2024-11-20T12:13:37,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN in 860 msec 2024-11-20T12:13:37,452 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:37,456 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 113951a1eeaf21f60b028a18a0a1a80b, server=843d83d51ba0,35591,1732104813307 because future has completed 2024-11-20T12:13:37,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=312c8b2fa94f26f46326bffa7ae2bcff, ASSIGN in 866 msec 2024-11-20T12:13:37,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=25 2024-11-20T12:13:37,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 113951a1eeaf21f60b028a18a0a1a80b, server=843d83d51ba0,35591,1732104813307 in 486 msec 2024-11-20T12:13:37,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, ASSIGN in 872 msec 2024-11-20T12:13:37,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=56, resume processing ppid=5 2024-11-20T12:13:37,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,35591,1732104813307 in 455 msec 2024-11-20T12:13:37,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN in 879 msec 2024-11-20T12:13:37,472 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=18 updating hbase:meta row=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:37,475 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=18, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0a3d655f8e46d7b0303a071be25a13da, server=843d83d51ba0,38659,1732104813354 because future has completed 2024-11-20T12:13:37,477 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=843d83d51ba0,35591,1732104813307, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T12:13:37,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=20 2024-11-20T12:13:37,480 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,35591,1732104813307 in 524 msec 2024-11-20T12:13:37,482 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN in 891 msec 2024-11-20T12:13:37,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=18 2024-11-20T12:13:37,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=18, state=SUCCESS, hasLock=false; OpenRegionProcedure 0a3d655f8e46d7b0303a071be25a13da, server=843d83d51ba0,38659,1732104813354 in 652 msec 2024-11-20T12:13:37,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=0a3d655f8e46d7b0303a071be25a13da, ASSIGN in 897 msec 2024-11-20T12:13:37,491 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, openSeqNum=2, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:37,492 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=843d83d51ba0,38659,1732104813354, table=test, region=fef27752c5c42f2f9602a38de6f56349. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-20T12:13:37,494 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=43, ppid=21, state=RUNNABLE, hasLock=false; OpenRegionProcedure fef27752c5c42f2f9602a38de6f56349, server=843d83d51ba0,38659,1732104813354 because future has completed 2024-11-20T12:13:37,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=21 2024-11-20T12:13:37,498 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=21, state=SUCCESS, hasLock=false; OpenRegionProcedure fef27752c5c42f2f9602a38de6f56349, server=843d83d51ba0,38659,1732104813354 in 625 msec 2024-11-20T12:13:37,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=4 2024-11-20T12:13:37,501 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=fef27752c5c42f2f9602a38de6f56349, ASSIGN in 908 msec 2024-11-20T12:13:37,503 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-20T12:13:37,503 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"test","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732104817503"}]},"ts":"1732104817503"} 2024-11-20T12:13:37,506 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=test, state=ENABLED in hbase:meta 2024-11-20T12:13:37,508 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=test execute state=CREATE_TABLE_POST_OPERATION 2024-11-20T12:13:37,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test in 1.8270 sec 2024-11-20T12:13:37,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-20T12:13:37,851 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:test completed 2024-11-20T12:13:37,851 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table test get assigned. Timeout = 60000ms 2024-11-20T12:13:37,854 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T12:13:37,878 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table test assigned to meta. Checking AM states. 2024-11-20T12:13:37,879 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T12:13:37,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table test assigned. 2024-11-20T12:13:37,880 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T12:13:37,885 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-20T12:13:37,910 DEBUG [Time-limited test {}] hbase.ClientMetaTableAccessor(255): Scanning META starting at row=test,, stopping at row=test ,, for max=2147483647 with caching=100 2024-11-20T12:13:37,927 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:13:37,929 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43862, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:13:37,931 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:13:37,933 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35868, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:13:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 99f9c663ffb10623454992ca39cafe6e changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8448ac25ea038ac8c63c9da8906d8673 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region abac5b1bb17e965219c5ca01a8ebc325 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0b4c46659583192ce01a652d7688d63d changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region f42c0bd7eed4146b0b4502022341fd1d changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 03e3b3cc4355d1bb26b41172ffed46ca changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 45e15b0320018bda387cb884d175aab6 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 113951a1eeaf21f60b028a18a0a1a80b changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 2186bb28c010fe9bd50ae06c405d542b changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 23c5a5a22a2d09813d1ff472bb76f407 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 325e5f934bae06dfb2390407a9e9726f changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region dec6f7c7f05d870bae6d81029b3676e2 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 0a3d655f8e46d7b0303a071be25a13da changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region fef27752c5c42f2f9602a38de6f56349 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 312c8b2fa94f26f46326bffa7ae2bcff changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c2a739d456605f6e8ff10b7be87ca995 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8978fd254a3dd3cd4570b36a1c18b8cb changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region ae055eeceea830ff461b6b4dc158461e changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 9b2ba9c7430b8a524b3966df315ad45c changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 6171c0ffde5383e6eb4635a4c76e3ad1 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region c7c42370d5f1944a9469b506921ac97c changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 633a6ec91d322a9170f4899c41489180 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 8deafbcc710170143f5159662c491d64 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 23f5434f0840c36b9d9bb2e81c743305 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 5f89ec3b6d6dbb62bc1c5a7ca523c08c changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 366d4b256c89d7860c4d7747ae8d7cc6 changed from -1.0 to 0.0, refreshing cache 2024-11-20T12:13:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] rsgroup.RSGroupBasedLoadBalancer(138): Balancing RSGroup=default 2024-11-20T12:13:37,986 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] rsgroup.RSGroupBasedLoadBalancer(151): Start Generate Balance plan for group: default 2024-11-20T12:13:37,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BaseLoadBalancer(619): Start Generate Balance plan for cluster. 2024-11-20T12:13:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BalancerClusterState(204): Hosts are {843d83d51ba0=0} racks are {/default-rack=0} 2024-11-20T12:13:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BalancerClusterState(303): server 0 has 10 regions 2024-11-20T12:13:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BalancerClusterState(303): server 1 has 8 regions 2024-11-20T12:13:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BalancerClusterState(303): server 2 has 9 regions 2024-11-20T12:13:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T12:13:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T12:13:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-20T12:13:37,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T12:13:37,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T12:13:37,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-20T12:13:37,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.BalancerClusterState(326): Number of tables=2, number of hosts=1, number of racks=1 2024-11-20T12:13:37,991 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.StochasticLoadBalancer(403): Cluster wide - Calculating plan. may take up to 30000ms to complete. 2024-11-20T12:13:37,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.StochasticLoadBalancer(515): Start StochasticLoadBalancer.balancer, initial weighted average imbalance=0.06931633435746164, functionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.06415002990995843, need balance); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.0); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); computedMaxSteps=64800 2024-11-20T12:13:38,486 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] balancer.StochasticLoadBalancer(562): Finished computing new moving plan. Computation took 497 ms to try 64800 different iterations. Found a solution that moves 1 regions; Going from a computed imbalance of 0.06931633435746164 to a new imbalance of 0.01690158032826896. funtionCost=RegionCountSkewCostFunction : (multiplier=500.0, imbalance=0.0); PrimaryRegionCountSkewCostFunction : (not needed); MoveCostFunction : (multiplier=7.0, imbalance=0.037037037037037035, need balance); ServerLocalityCostFunction : (multiplier=25.0, imbalance=0.0); RackLocalityCostFunction : (multiplier=15.0, imbalance=0.0); TableSkewCostFunction : (multiplier=35.0, imbalance=0.0); RegionReplicaHostCostFunction : (not needed); RegionReplicaRackCostFunction : (not needed); ReadRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); CPRequestCostFunction : (multiplier=5.0, imbalance=0.0); WriteRequestCostFunction : (multiplier=5.0, imbalance=1.0, need balance); MemStoreSizeCostFunction : (multiplier=5.0, imbalance=0.0); StoreFileCostFunction : (multiplier=5.0, imbalance=0.0); 2024-11-20T12:13:38,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.HMaster(2167): Balancer plans size is 1, the balance interval is 300000 ms, and the max number regions in transition is 27 2024-11-20T12:13:38,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.HMaster(2172): balance hri=113951a1eeaf21f60b028a18a0a1a80b, source=843d83d51ba0,35591,1732104813307, destination=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:38,491 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:38,493 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=113951a1eeaf21f60b028a18a0a1a80b, regionState=CLOSING, regionLocation=843d83d51ba0,35591,1732104813307 2024-11-20T12:13:38,495 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE because future has completed 2024-11-20T12:13:38,496 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-20T12:13:38,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 113951a1eeaf21f60b028a18a0a1a80b, server=843d83d51ba0,35591,1732104813307}] 2024-11-20T12:13:38,653 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(122): Close 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:38,654 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-20T12:13:38,654 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1722): Closing 113951a1eeaf21f60b028a18a0a1a80b, disabling compactions & flushes 2024-11-20T12:13:38,654 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1755): Closing region test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:38,654 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:38,655 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. after waiting 0 ms 2024-11-20T12:13:38,655 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:38,661 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/113951a1eeaf21f60b028a18a0a1a80b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:38,664 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1973): Closed test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:38,664 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegion(1676): Region close journal for 113951a1eeaf21f60b028a18a0a1a80b: Waiting for close lock at 1732104818654Running coprocessor pre-close hooks at 1732104818654Disabling compacts and flushes for region at 1732104818654Disabling writes for close at 1732104818655 (+1 ms)Writing region close event to WAL at 1732104818655Running coprocessor post-close hooks at 1732104818662 (+7 ms)Closed at 1732104818664 (+2 ms) 2024-11-20T12:13:38,665 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] regionserver.HRegionServer(3302): Adding 113951a1eeaf21f60b028a18a0a1a80b move to 843d83d51ba0,38659,1732104813354 record at close sequenceid=2 2024-11-20T12:13:38,669 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION, pid=58}] handler.UnassignRegionHandler(157): Closed 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:38,670 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=113951a1eeaf21f60b028a18a0a1a80b, regionState=CLOSED 2024-11-20T12:13:38,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE, hasLock=false; CloseRegionProcedure 113951a1eeaf21f60b028a18a0a1a80b, server=843d83d51ba0,35591,1732104813307 because future has completed 2024-11-20T12:13:38,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-20T12:13:38,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; CloseRegionProcedure 113951a1eeaf21f60b028a18a0a1a80b, server=843d83d51ba0,35591,1732104813307 in 177 msec 2024-11-20T12:13:38,677 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE; state=CLOSED, location=843d83d51ba0,38659,1732104813354; forceNewPlan=false, retain=false 2024-11-20T12:13:38,828 INFO [843d83d51ba0:39157 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-20T12:13:38,828 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:38,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE because future has completed 2024-11-20T12:13:38,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 113951a1eeaf21f60b028a18a0a1a80b, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:38,993 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:38,993 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 113951a1eeaf21f60b028a18a0a1a80b, NAME => 'test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b.', STARTKEY => 'uuu', ENDKEY => 'vvv'} 2024-11-20T12:13:38,994 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:38,994 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:38,994 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:38,994 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:38,996 INFO [StoreOpener-113951a1eeaf21f60b028a18a0a1a80b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:38,998 INFO [StoreOpener-113951a1eeaf21f60b028a18a0a1a80b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 113951a1eeaf21f60b028a18a0a1a80b columnFamilyName family 2024-11-20T12:13:38,998 DEBUG [StoreOpener-113951a1eeaf21f60b028a18a0a1a80b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:38,998 INFO [StoreOpener-113951a1eeaf21f60b028a18a0a1a80b-1 {}] regionserver.HStore(327): Store=113951a1eeaf21f60b028a18a0a1a80b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:38,999 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:39,000 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:39,001 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:39,002 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:39,002 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:39,004 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:39,005 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 113951a1eeaf21f60b028a18a0a1a80b; next sequenceid=5; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59609769, jitterRate=-0.11174522340297699}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:39,005 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:39,005 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 113951a1eeaf21f60b028a18a0a1a80b: Running coprocessor pre-open hook at 1732104818994Writing region info on filesystem at 1732104818994Initializing all the Stores at 1732104818996 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104818996Cleaning up temporary data from old regions at 1732104819002 (+6 ms)Running coprocessor post-open hooks at 1732104819005 (+3 ms)Region opened successfully at 1732104819005 2024-11-20T12:13:39,007 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b., pid=59, masterSystemTime=1732104818985 2024-11-20T12:13:39,010 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:39,010 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:39,010 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=57 updating hbase:meta row=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, openSeqNum=5, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:39,013 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=57, state=RUNNABLE, hasLock=false; OpenRegionProcedure 113951a1eeaf21f60b028a18a0a1a80b, server=843d83d51ba0,38659,1732104813354 because future has completed 2024-11-20T12:13:39,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=57 2024-11-20T12:13:39,017 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=57, state=SUCCESS, hasLock=false; OpenRegionProcedure 113951a1eeaf21f60b028a18a0a1a80b, server=843d83d51ba0,38659,1732104813354 in 182 msec 2024-11-20T12:13:39,019 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE in 528 msec 2024-11-20T12:13:39,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.HMaster(2203): Balancer is going into sleep until next period in 300000ms 2024-11-20T12:13:39,093 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [900,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T12:13:39,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.MasterRpcServices(567): Client=jenkins//172.17.0.3 set balanceSwitch=false 2024-11-20T12:13:39,115 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestRollbackSCP#testFailAndRollback Thread=377, OpenFileDescriptor=623, MaxFileDescriptor=1048576, SystemLoadAverage=598, ProcessCount=11, AvailableMemoryMB=1676 2024-11-20T12:13:39,118 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2234): Found 1 regions for table hbase:meta 2024-11-20T12:13:39,119 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(2240): firstRegionName=hbase:meta,,1 2024-11-20T12:13:39,119 INFO [Time-limited test {}] hbase.SingleProcessHBaseCluster(267): Killing 843d83d51ba0,35591,1732104813307 2024-11-20T12:13:39,119 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 843d83d51ba0,35591,1732104813307: Simulated kill ***** 2024-11-20T12:13:39,120 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-20T12:13:39,121 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-20T12:13:39,123 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-20T12:13:39,123 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-20T12:13:39,124 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-20T12:13:39,124 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 297292880 }, "NonHeapMemoryUsage": { "committed": 153681920, "init": 7667712, "max": -1, "used": 151437528 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "843d83d51ba0" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "843d83d51ba0" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.Context": "regionserver", "tag.Hostname": "843d83d51ba0" } ] } 2024-11-20T12:13:39,127 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=39157 {}] master.MasterRpcServices(700): 843d83d51ba0,35591,1732104813307 reported a fatal error: ***** ABORTING region server 843d83d51ba0,35591,1732104813307: Simulated kill ***** 2024-11-20T12:13:39,129 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '843d83d51ba0,35591,1732104813307' ***** 2024-11-20T12:13:39,129 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Simulated kill 2024-11-20T12:13:39,129 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T12:13:39,130 INFO [RS:1;843d83d51ba0:35591 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-20T12:13:39,130 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T12:13:39,130 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [15,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T12:13:39,130 INFO [RS:1;843d83d51ba0:35591 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-20T12:13:39,130 INFO [RS:1;843d83d51ba0:35591 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T12:13:39,130 INFO [RS:1;843d83d51ba0:35591 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;843d83d51ba0:35591. 2024-11-20T12:13:39,130 DEBUG [RS:1;843d83d51ba0:35591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:13:39,136 DEBUG [RS:1;843d83d51ba0:35591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:13:39,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741836_1012 (size=33749) 2024-11-20T12:13:39,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741836_1012 (size=33749) 2024-11-20T12:13:39,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741836_1012 (size=33749) 2024-11-20T12:13:39,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741834_1010 (size=2769) 2024-11-20T12:13:39,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741834_1010 (size=2769) 2024-11-20T12:13:39,148 DEBUG [RS:1;843d83d51ba0:35591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:13:39,149 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T12:13:39,149 INFO [RS:1;843d83d51ba0:35591 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35591 2024-11-20T12:13:39,210 INFO [regionserver/843d83d51ba0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T12:13:39,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T12:13:39,225 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/843d83d51ba0,35591,1732104813307 2024-11-20T12:13:39,225 INFO [RS:1;843d83d51ba0:35591 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T12:13:39,236 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [843d83d51ba0,35591,1732104813307] 2024-11-20T12:13:39,245 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/843d83d51ba0,35591,1732104813307 already deleted, retry=false 2024-11-20T12:13:39,246 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 843d83d51ba0,35591,1732104813307 on 843d83d51ba0,39157,1732104812301 2024-11-20T12:13:39,253 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:39,259 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=60 for 843d83d51ba0,35591,1732104813307 (carryingMeta=true) 843d83d51ba0,35591,1732104813307/CRASHED/regionCount=9/lock=java.util.concurrent.locks.ReentrantReadWriteLock@6b332840[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-20T12:13:39,260 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=60, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:39,261 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true, isMeta: true 2024-11-20T12:13:39,264 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting 2024-11-20T12:13:39,266 INFO [PEWorker-1 {}] master.SplitWALManager(105): 843d83d51ba0,35591,1732104813307 WAL count=1, meta=true 2024-11-20T12:13:39,269 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta}] 2024-11-20T12:13:39,277 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:39,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:39,332 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [30,000] milli-secs(wait.for.ratio=[1]) 2024-11-20T12:13:39,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T12:13:39,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35591-0x1015829ee370002, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T12:13:39,336 INFO [RS:1;843d83d51ba0:35591 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T12:13:39,336 INFO [RS:1;843d83d51ba0:35591 {}] regionserver.HRegionServer(1031): Exiting; stopping=843d83d51ba0,35591,1732104813307; zookeeper connection closed. 2024-11-20T12:13:39,337 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5136e179 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5136e179 2024-11-20T12:13:39,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38659 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=62 2024-11-20T12:13:39,469 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(299): Splitting hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, size=33.0 K (33749bytes) 2024-11-20T12:13:39,470 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-20T12:13:39,471 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-20T12:13:39,471 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta 2024-11-20T12:13:39,477 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta after 4ms 2024-11-20T12:13:39,485 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(310): Open hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta took 17ms 2024-11-20T12:13:39,494 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-20T12:13:39,501 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta.temp 2024-11-20T12:13:39,504 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta.temp 2024-11-20T12:13:39,506 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta so closing down 2024-11-20T12:13:39,506 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T12:13:39,510 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T12:13:39,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741864_1040 (size=33508) 2024-11-20T12:13:39,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741864_1040 (size=33508) 2024-11-20T12:13:39,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741864_1040 (size=33508) 2024-11-20T12:13:39,528 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) 2024-11-20T12:13:39,531 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta.temp to hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-20T12:13:39,534 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(425): Processed 62 edits across 1 Regions in 45 ms; skipped=1; WAL=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false 2024-11-20T12:13:39,534 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] wal.WALSplitter(428): Completed split of hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, journal: Splitting hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, size=33.0 K (33749bytes) at 1732104819470Creating recovered edits writer path=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta.temp at 1732104819504 (+34 ms)Finishing writing output for hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta so closing down at 1732104819506 (+2 ms)3 split writer threads finished at 1732104819510 (+4 ms)Closed recovered edits writer path=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta.temp (wrote 61 edits, skipped 0 edits in 4 ms) at 1732104819528 (+18 ms)Rename recovered edits hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta.temp to hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 at 1732104819531 (+3 ms)Processed 62 edits across 1 Regions in 45 ms; skipped=1; WAL=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, size=33.0 K, length=33749, corrupted=false, cancelled=false at 1732104819534 (+3 ms) 2024-11-20T12:13:39,535 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta 2024-11-20T12:13:39,536 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-0 {event_type=RS_LOG_REPLAY, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-20T12:13:39,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-11-20T12:13:39,545 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta to hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/oldWALs 2024-11-20T12:13:39,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-11-20T12:13:39,547 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 in 265 msec 2024-11-20T12:13:39,549 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:39,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-20T12:13:39,552 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 in 281 msec 2024-11-20T12:13:39,555 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting dir is empty, no logs to split. 2024-11-20T12:13:39,555 INFO [PEWorker-1 {}] master.SplitWALManager(105): 843d83d51ba0,35591,1732104813307 WAL count=0, meta=true 2024-11-20T12:13:39,555 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 843d83d51ba0,35591,1732104813307 WAL splitting is done? wals=0, meta=true 2024-11-20T12:13:39,557 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-20T12:13:39,560 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:39,561 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T12:13:39,711 DEBUG [843d83d51ba0:39157 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-20T12:13:39,712 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(204): Hosts are {843d83d51ba0=0} racks are {/default-rack=0} 2024-11-20T12:13:39,712 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T12:13:39,712 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T12:13:39,712 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T12:13:39,712 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T12:13:39,712 INFO [843d83d51ba0:39157 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T12:13:39,712 INFO [843d83d51ba0:39157 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T12:13:39,712 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T12:13:39,713 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:39,714 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPENING 2024-11-20T12:13:39,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:39,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:39,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:39,737 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=63, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:39,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:39,737 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:39,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:39,738 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:39,745 INFO [regionserver/843d83d51ba0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-20T12:13:39,746 INFO [regionserver/843d83d51ba0:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-20T12:13:39,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741841_1017 (size=45) 2024-11-20T12:13:39,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741843_1019 (size=45) 2024-11-20T12:13:39,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741845_1021 (size=45) 2024-11-20T12:13:39,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741842_1018 (size=45) 2024-11-20T12:13:39,897 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-20T12:13:39,897 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-20T12:13:39,898 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-20T12:13:39,901 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=843d83d51ba0%2C38659%2C1732104813354.meta, suffix=.meta, logDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,38659,1732104813354, archiveDir=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/oldWALs, maxLogs=32 2024-11-20T12:13:39,918 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,38659,1732104813354/843d83d51ba0%2C38659%2C1732104813354.meta.1732104819903.meta, exclude list is [], retry=0 2024-11-20T12:13:39,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41703,DS-9ec5aa7e-1c59-486c-ad94-94a00f2e2c4d,DISK] 2024-11-20T12:13:39,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38431,DS-3793a777-6860-453f-b38c-f277d5ad850e,DISK] 2024-11-20T12:13:39,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34249,DS-924c6f71-12d9-407c-a800-6de4979baefb,DISK] 2024-11-20T12:13:39,929 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,38659,1732104813354/843d83d51ba0%2C38659%2C1732104813354.meta.1732104819903.meta 2024-11-20T12:13:39,929 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43069:43069),(127.0.0.1/127.0.0.1:38745:38745),(127.0.0.1/127.0.0.1:41027:41027)] 2024-11-20T12:13:39,930 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-20T12:13:39,930 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-20T12:13:39,930 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-20T12:13:39,931 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-20T12:13:39,931 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-20T12:13:39,931 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:39,931 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-20T12:13:39,931 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-20T12:13:39,934 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-20T12:13:39,935 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-20T12:13:39,935 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:39,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:39,936 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-20T12:13:39,938 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-20T12:13:39,938 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:39,938 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:39,939 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-20T12:13:39,940 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-20T12:13:39,940 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:39,940 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:39,941 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-20T12:13:39,941 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-20T12:13:39,942 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:39,942 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-20T12:13:39,942 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-20T12:13:39,943 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740 2024-11-20T12:13:39,945 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740 2024-11-20T12:13:39,947 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-20T12:13:39,962 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5793): Applied 277, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=64, path=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-20T12:13:39,965 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=29.41 KB heapSize=50.67 KB 2024-11-20T12:13:40,028 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/info/1a832283fa604750ae60a083710ea523 is 138, key is test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407./info:regioninfo/1732104817379/Put/seqid=0 2024-11-20T12:13:40,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741866_1042 (size=36371) 2024-11-20T12:13:40,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741866_1042 (size=36371) 2024-11-20T12:13:40,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741866_1042 (size=36371) 2024-11-20T12:13:40,044 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.26 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/info/1a832283fa604750ae60a083710ea523 2024-11-20T12:13:40,103 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/ns/689c412163a34a0bb6a5c697bb945b62 is 43, key is default/ns:d/1732104815484/Put/seqid=0 2024-11-20T12:13:40,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741867_1043 (size=5153) 2024-11-20T12:13:40,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741867_1043 (size=5153) 2024-11-20T12:13:40,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741867_1043 (size=5153) 2024-11-20T12:13:40,113 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/ns/689c412163a34a0bb6a5c697bb945b62 2024-11-20T12:13:40,137 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/table/6cd40cca987c4ccca0824cd6313ced59 is 40, key is test/table:state/1732104817503/Put/seqid=0 2024-11-20T12:13:40,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741868_1044 (size=5165) 2024-11-20T12:13:40,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741868_1044 (size=5165) 2024-11-20T12:13:40,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741868_1044 (size=5165) 2024-11-20T12:13:40,149 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=72 B at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/table/6cd40cca987c4ccca0824cd6313ced59 2024-11-20T12:13:40,160 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/info/1a832283fa604750ae60a083710ea523 as hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/info/1a832283fa604750ae60a083710ea523 2024-11-20T12:13:40,170 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/info/1a832283fa604750ae60a083710ea523, entries=273, sequenceid=64, filesize=35.5 K 2024-11-20T12:13:40,174 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/ns/689c412163a34a0bb6a5c697bb945b62 as hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/ns/689c412163a34a0bb6a5c697bb945b62 2024-11-20T12:13:40,184 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/ns/689c412163a34a0bb6a5c697bb945b62, entries=2, sequenceid=64, filesize=5.0 K 2024-11-20T12:13:40,186 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/table/6cd40cca987c4ccca0824cd6313ced59 as hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/table/6cd40cca987c4ccca0824cd6313ced59 2024-11-20T12:13:40,197 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/table/6cd40cca987c4ccca0824cd6313ced59, entries=2, sequenceid=64, filesize=5.0 K 2024-11-20T12:13:40,203 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(3140): Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 234ms, sequenceid=64, compaction requested=false; wal=null 2024-11-20T12:13:40,204 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-20T12:13:40,207 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/0000000000000000064 2024-11-20T12:13:40,209 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-20T12:13:40,209 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-20T12:13:40,210 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-20T12:13:40,213 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-20T12:13:40,216 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/64.seqid, newMaxSeqId=64, maxSeqId=1 2024-11-20T12:13:40,217 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=65; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59800963, jitterRate=-0.10889621078968048}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-20T12:13:40,217 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-20T12:13:40,218 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732104819932Writing region info on filesystem at 1732104819932Initializing all the Stores at 1732104819933 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104819933Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104819934 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104819934Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732104819934Obtaining lock to block concurrent updates at 1732104819965 (+31 ms)Preparing flush snapshotting stores in 1588230740 at 1732104819965Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=30111, getHeapSize=51824, getOffHeapSize=0, getCellsCount=277 at 1732104819968 (+3 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732104819968Flushing 1588230740/info: creating writer at 1732104819970 (+2 ms)Flushing 1588230740/info: appending metadata at 1732104820018 (+48 ms)Flushing 1588230740/info: closing flushed file at 1732104820021 (+3 ms)Flushing 1588230740/ns: creating writer at 1732104820088 (+67 ms)Flushing 1588230740/ns: appending metadata at 1732104820102 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1732104820102Flushing 1588230740/table: creating writer at 1732104820120 (+18 ms)Flushing 1588230740/table: appending metadata at 1732104820135 (+15 ms)Flushing 1588230740/table: closing flushed file at 1732104820135Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ca31eac: reopening flushed file at 1732104820158 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5d7e96ad: reopening flushed file at 1732104820172 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4fb5c45: reopening flushed file at 1732104820185 (+13 ms)Finished flush of dataSize ~29.41 KB/30111, heapSize ~50.38 KB/51584, currentSize=0 B/0 for 1588230740 in 234ms, sequenceid=64, compaction requested=false; wal=null at 1732104820203 (+18 ms)Cleaning up temporary data from old regions at 1732104820209 (+6 ms)Running coprocessor post-open hooks at 1732104820217 (+8 ms)Region opened successfully at 1732104820218 (+1 ms) 2024-11-20T12:13:40,219 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=64, masterSystemTime=1732104819890 2024-11-20T12:13:40,222 DEBUG [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-20T12:13:40,222 INFO [RS_OPEN_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_META, pid=64}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-20T12:13:40,223 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=63 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=65, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:40,223 INFO [PEWorker-1 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:40,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:40,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:40,277 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:40,277 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=64, ppid=63, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:40,277 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:40,277 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:40,277 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:40,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-20T12:13:40,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 in 540 msec 2024-11-20T12:13:40,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=60 2024-11-20T12:13:40,283 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 723 msec 2024-11-20T12:13:40,283 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 843d83d51ba0,35591,1732104813307 had 9 regions 2024-11-20T12:13:40,285 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=60, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true, isMeta: false 2024-11-20T12:13:40,287 INFO [PEWorker-1 {}] master.SplitWALManager(105): 843d83d51ba0,35591,1732104813307 WAL count=1, meta=false 2024-11-20T12:13:40,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=60, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865}] 2024-11-20T12:13:40,290 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:40,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:40,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=38659 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=66 2024-11-20T12:13:40,461 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(299): Splitting hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865, size=2.7 K (2769bytes) 2024-11-20T12:13:40,461 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865 2024-11-20T12:13:40,461 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865 after 0ms 2024-11-20T12:13:40,464 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(310): Open hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865 took 4ms 2024-11-20T12:13:40,468 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 03e3b3cc4355d1bb26b41172ffed46ca: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T12:13:40,471 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 99f9c663ffb10623454992ca39cafe6e: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T12:13:40,473 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 2186bb28c010fe9bd50ae06c405d542b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T12:13:40,475 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for abac5b1bb17e965219c5ca01a8ebc325: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T12:13:40,477 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for f42c0bd7eed4146b0b4502022341fd1d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T12:13:40,479 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 8448ac25ea038ac8c63c9da8906d8673: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T12:13:40,480 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 45e15b0320018bda387cb884d175aab6: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T12:13:40,482 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 0b4c46659583192ce01a652d7688d63d: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T12:13:40,483 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(352): Last flushed sequenceid for 113951a1eeaf21f60b028a18a0a1a80b: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "family" sequence_id: 18446744073709551615 } 2024-11-20T12:13:40,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865 so closing down 2024-11-20T12:13:40,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-20T12:13:40,484 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.OutputSink(145): 3 split writer threads finished 2024-11-20T12:13:40,484 INFO [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(425): Processed 10 edits across 0 Regions in 19 ms; skipped=10; WAL=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865, size=2.7 K, length=2769, corrupted=false, cancelled=false 2024-11-20T12:13:40,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] wal.WALSplitter(428): Completed split of hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865, journal: Splitting hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865, size=2.7 K (2769bytes) at 1732104820461Finishing writing output for hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865 so closing down at 1732104820484 (+23 ms)3 split writer threads finished at 1732104820484Processed 10 edits across 0 Regions in 19 ms; skipped=10; WAL=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865, size=2.7 K, length=2769, corrupted=false, cancelled=false at 1732104820484 2024-11-20T12:13:40,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865 2024-11-20T12:13:40,484 DEBUG [RS_LOG_REPLAY_OPS-regionserver/843d83d51ba0:0-1 {event_type=RS_LOG_REPLAY, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-20T12:13:40,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39157 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-11-20T12:13:40,490 INFO [PEWorker-1 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting/843d83d51ba0%2C35591%2C1732104813307.1732104814865 to hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/oldWALs 2024-11-20T12:13:40,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-11-20T12:13:40,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 in 199 msec 2024-11-20T12:13:40,494 DEBUG [PEWorker-1 {}] master.SplitWALManager(172): Release split WAL worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:40,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=60 2024-11-20T12:13:40,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 in 207 msec 2024-11-20T12:13:40,499 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/WALs/843d83d51ba0,35591,1732104813307-splitting dir is empty, no logs to split. 2024-11-20T12:13:40,499 INFO [PEWorker-1 {}] master.SplitWALManager(105): 843d83d51ba0,35591,1732104813307 WAL count=0, meta=false 2024-11-20T12:13:40,499 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 843d83d51ba0,35591,1732104813307 WAL splitting is done? wals=0, meta=false 2024-11-20T12:13:40,502 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN}, {pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN}, {pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN}, {pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN}, {pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN}, {pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN}, {pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN}, {pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN}] 2024-11-20T12:13:40,503 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:40,504 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T12:13:40,505 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:40,506 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T12:13:40,507 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:40,508 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T12:13:40,509 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:40,510 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T12:13:40,512 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:40,512 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T12:13:40,513 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:40,514 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T12:13:40,515 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:40,515 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T12:13:40,516 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:40,517 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-20T12:13:40,655 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(204): Hosts are {843d83d51ba0=0} racks are {/default-rack=0} 2024-11-20T12:13:40,655 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-20T12:13:40,655 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-20T12:13:40,655 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-20T12:13:40,655 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-20T12:13:40,655 INFO [843d83d51ba0:39157 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-20T12:13:40,656 INFO [843d83d51ba0:39157 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-20T12:13:40,656 DEBUG [843d83d51ba0:39157 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-20T12:13:40,656 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=71 updating hbase:meta row=45e15b0320018bda387cb884d175aab6, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:40,657 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=73 updating hbase:meta row=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:40,657 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:40,658 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=67 updating hbase:meta row=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, regionLocation=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:40,657 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 843d83d51ba0:35591 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:13:40,658 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=72 updating hbase:meta row=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:40,659 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:40,659 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=69 updating hbase:meta row=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:40,659 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1, error=java.net.ConnectException: Call to address=843d83d51ba0:35591 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 2024-11-20T12:13:40,660 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, regionLocation=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:40,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-20T12:13:40,660 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1 from cache 2024-11-20T12:13:40,661 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 843d83d51ba0:35591 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 2024-11-20T12:13:40,661 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 843d83d51ba0:35591 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-20T12:13:40,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=843d83d51ba0:35591 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 2024-11-20T12:13:40,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=843d83d51ba0:35591 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 2024-11-20T12:13:40,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=843d83d51ba0:35591 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 2024-11-20T12:13:40,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=843d83d51ba0:35591 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 2024-11-20T12:13:40,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=843d83d51ba0:35591 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 2024-11-20T12:13:40,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=843d83d51ba0:35591 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 2024-11-20T12:13:40,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,35591,1732104813307, seqNum=-1 , the old value is null, error=java.net.ConnectException: Call to address=843d83d51ba0:35591 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 2024-11-20T12:13:40,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 843d83d51ba0:35591 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 843d83d51ba0/172.17.0.3:35591 2024-11-20T12:13:40,767 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-20T12:13:40,767 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=843d83d51ba0,38659,1732104813354, seqNum=-1] 2024-11-20T12:13:40,767 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-20T12:13:40,769 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45367, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-20T12:13:40,771 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN because future has completed 2024-11-20T12:13:40,771 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:40,772 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN because future has completed 2024-11-20T12:13:40,773 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN because future has completed 2024-11-20T12:13:40,775 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN because future has completed 2024-11-20T12:13:40,776 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN because future has completed 2024-11-20T12:13:40,777 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:40,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN because future has completed 2024-11-20T12:13:40,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN because future has completed 2024-11-20T12:13:40,779 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN because future has completed 2024-11-20T12:13:40,780 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:40,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:40,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=67, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:40,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,44663,1732104813203}] 2024-11-20T12:13:40,790 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8448ac25ea038ac8c63c9da8906d8673, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:40,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b4c46659583192ce01a652d7688d63d, server=843d83d51ba0,38659,1732104813354}] 2024-11-20T12:13:40,872 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-20T12:13:40,930 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:40,930 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'test' 2024-11-20T12:13:40,930 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => 99f9c663ffb10623454992ca39cafe6e, NAME => 'test,,1732104815671.99f9c663ffb10623454992ca39cafe6e.', STARTKEY => '', ENDKEY => 'bbb'} 2024-11-20T12:13:40,931 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:40,931 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated test,,1732104815671.99f9c663ffb10623454992ca39cafe6e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:40,931 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:40,931 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:40,933 INFO [StoreOpener-99f9c663ffb10623454992ca39cafe6e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:40,934 INFO [StoreOpener-99f9c663ffb10623454992ca39cafe6e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99f9c663ffb10623454992ca39cafe6e columnFamilyName family 2024-11-20T12:13:40,935 DEBUG [StoreOpener-99f9c663ffb10623454992ca39cafe6e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:40,936 INFO [StoreOpener-99f9c663ffb10623454992ca39cafe6e-1 {}] regionserver.HStore(327): Store=99f9c663ffb10623454992ca39cafe6e/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:40,936 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(132): Open test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:40,936 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:40,936 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7752): Opening region: {ENCODED => 0b4c46659583192ce01a652d7688d63d, NAME => 'test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d.', STARTKEY => 'jjj', ENDKEY => 'kkk'} 2024-11-20T12:13:40,936 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:40,936 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(898): Instantiated test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:40,937 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7794): checking encryption for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:40,937 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:40,937 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(7797): checking classloading for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:40,938 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:40,939 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:40,939 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:40,939 INFO [StoreOpener-0b4c46659583192ce01a652d7688d63d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:40,940 INFO [StoreOpener-0b4c46659583192ce01a652d7688d63d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0b4c46659583192ce01a652d7688d63d columnFamilyName family 2024-11-20T12:13:40,940 DEBUG [StoreOpener-0b4c46659583192ce01a652d7688d63d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:40,940 INFO [StoreOpener-0b4c46659583192ce01a652d7688d63d-1 {}] regionserver.HStore(327): Store=0b4c46659583192ce01a652d7688d63d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:40,941 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1038): replaying wal for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:40,941 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:40,941 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:40,942 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened 99f9c663ffb10623454992ca39cafe6e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67979224, jitterRate=0.012969374656677246}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:40,942 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:40,942 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for 99f9c663ffb10623454992ca39cafe6e: Running coprocessor pre-open hook at 1732104820931Writing region info on filesystem at 1732104820931Initializing all the Stores at 1732104820933 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104820933Cleaning up temporary data from old regions at 1732104820939 (+6 ms)Running coprocessor post-open hooks at 1732104820942 (+3 ms)Region opened successfully at 1732104820942 2024-11-20T12:13:40,943 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:40,943 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1048): stopping wal replay for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:40,943 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1060): Cleaning up temporary data for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:40,943 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for test,,1732104815671.99f9c663ffb10623454992ca39cafe6e., pid=79, masterSystemTime=1732104820924 2024-11-20T12:13:40,946 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1093): writing seq id for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:40,946 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:40,946 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:40,946 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:40,946 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => f42c0bd7eed4146b0b4502022341fd1d, NAME => 'test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d.', STARTKEY => 'mmm', ENDKEY => 'nnn'} 2024-11-20T12:13:40,947 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:40,947 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:40,947 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:40,947 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:40,947 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1114): Opened 0b4c46659583192ce01a652d7688d63d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75247944, jitterRate=0.12128174304962158}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:40,947 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:40,947 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegion(1006): Region open journal for 0b4c46659583192ce01a652d7688d63d: Running coprocessor pre-open hook at 1732104820937Writing region info on filesystem at 1732104820937Initializing all the Stores at 1732104820938 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104820938Cleaning up temporary data from old regions at 1732104820943 (+5 ms)Running coprocessor post-open hooks at 1732104820947 (+4 ms)Region opened successfully at 1732104820947 2024-11-20T12:13:40,948 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2236): Post open deploy tasks for test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d., pid=82, masterSystemTime=1732104820931 2024-11-20T12:13:40,949 WARN [PEWorker-1 {}] procedure2.ProcedureTestingUtility(201): Set Kill before store update in rollback to: true 2024-11-20T12:13:40,949 DEBUG [PEWorker-1 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:40,950 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] regionserver.HRegionServer(2266): Finished post open deploy task for test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:40,951 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=82}] handler.AssignRegionHandler(153): Opened test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:40,951 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(132): Open test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:40,951 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7752): Opening region: {ENCODED => 2186bb28c010fe9bd50ae06c405d542b, NAME => 'test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b.', STARTKEY => 'xxx', ENDKEY => 'yyy'} 2024-11-20T12:13:40,950 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:40,951 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:40,951 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(898): Instantiated test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:40,952 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7794): checking encryption for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:40,952 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(7797): checking classloading for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:40,953 INFO [StoreOpener-f42c0bd7eed4146b0b4502022341fd1d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:40,954 INFO [StoreOpener-2186bb28c010fe9bd50ae06c405d542b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:40,955 INFO [StoreOpener-2186bb28c010fe9bd50ae06c405d542b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2186bb28c010fe9bd50ae06c405d542b columnFamilyName family 2024-11-20T12:13:40,955 INFO [StoreOpener-f42c0bd7eed4146b0b4502022341fd1d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f42c0bd7eed4146b0b4502022341fd1d columnFamilyName family 2024-11-20T12:13:40,955 DEBUG [StoreOpener-2186bb28c010fe9bd50ae06c405d542b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:40,955 DEBUG [StoreOpener-f42c0bd7eed4146b0b4502022341fd1d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:40,955 INFO [StoreOpener-f42c0bd7eed4146b0b4502022341fd1d-1 {}] regionserver.HStore(327): Store=f42c0bd7eed4146b0b4502022341fd1d/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:40,955 INFO [StoreOpener-2186bb28c010fe9bd50ae06c405d542b-1 {}] regionserver.HStore(327): Store=2186bb28c010fe9bd50ae06c405d542b/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:40,956 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:40,956 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1038): replaying wal for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:40,956 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:40,957 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:40,958 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:40,958 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:40,958 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:40,959 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:40,959 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1048): stopping wal replay for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:40,959 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1060): Cleaning up temporary data for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:40,961 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:40,962 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1093): writing seq id for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:40,963 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened f42c0bd7eed4146b0b4502022341fd1d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64371530, jitterRate=-0.04078945517539978}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:40,963 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:40,963 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for f42c0bd7eed4146b0b4502022341fd1d: Running coprocessor pre-open hook at 1732104820947Writing region info on filesystem at 1732104820947Initializing all the Stores at 1732104820953 (+6 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104820953Cleaning up temporary data from old regions at 1732104820958 (+5 ms)Running coprocessor post-open hooks at 1732104820963 (+5 ms)Region opened successfully at 1732104820963 2024-11-20T12:13:40,963 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1114): Opened 2186bb28c010fe9bd50ae06c405d542b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72271055, jitterRate=0.07692264020442963}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:40,964 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:40,964 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d., pid=80, masterSystemTime=1732104820924 2024-11-20T12:13:40,964 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegion(1006): Region open journal for 2186bb28c010fe9bd50ae06c405d542b: Running coprocessor pre-open hook at 1732104820952Writing region info on filesystem at 1732104820952Initializing all the Stores at 1732104820953 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104820953Cleaning up temporary data from old regions at 1732104820959 (+6 ms)Running coprocessor post-open hooks at 1732104820964 (+5 ms)Region opened successfully at 1732104820964 2024-11-20T12:13:40,962 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=true, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-20T12:13:40,965 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:40,966 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b4c46659583192ce01a652d7688d63d, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:40,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:40,967 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2236): Post open deploy tasks for test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b., pid=78, masterSystemTime=1732104820931 2024-11-20T12:13:40,967 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b4c46659583192ce01a652d7688d63d, server=843d83d51ba0,38659,1732104813354 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:40,967 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:40,967 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:40,968 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:40,968 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 45e15b0320018bda387cb884d175aab6, NAME => 'test,sss,1732104815671.45e15b0320018bda387cb884d175aab6.', STARTKEY => 'sss', ENDKEY => 'ttt'} 2024-11-20T12:13:40,968 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:40,968 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated test,sss,1732104815671.45e15b0320018bda387cb884d175aab6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:40,968 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:40,968 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:40,970 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] regionserver.HRegionServer(2266): Finished post open deploy task for test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:40,970 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=78}] handler.AssignRegionHandler(153): Opened test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:40,970 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(132): Open test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:40,970 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7752): Opening region: {ENCODED => 8448ac25ea038ac8c63c9da8906d8673, NAME => 'test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673.', STARTKEY => 'eee', ENDKEY => 'fff'} 2024-11-20T12:13:40,970 INFO [StoreOpener-45e15b0320018bda387cb884d175aab6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:40,971 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:40,971 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(898): Instantiated test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:40,971 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7794): checking encryption for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:40,971 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(7797): checking classloading for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:40,973 INFO [StoreOpener-45e15b0320018bda387cb884d175aab6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 45e15b0320018bda387cb884d175aab6 columnFamilyName family 2024-11-20T12:13:40,973 DEBUG [StoreOpener-45e15b0320018bda387cb884d175aab6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:40,974 INFO [StoreOpener-8448ac25ea038ac8c63c9da8906d8673-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:40,974 INFO [StoreOpener-45e15b0320018bda387cb884d175aab6-1 {}] regionserver.HStore(327): Store=45e15b0320018bda387cb884d175aab6/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:40,974 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:40,975 INFO [StoreOpener-8448ac25ea038ac8c63c9da8906d8673-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8448ac25ea038ac8c63c9da8906d8673 columnFamilyName family 2024-11-20T12:13:40,975 DEBUG [StoreOpener-8448ac25ea038ac8c63c9da8906d8673-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:40,975 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:40,975 INFO [StoreOpener-8448ac25ea038ac8c63c9da8906d8673-1 {}] regionserver.HStore(327): Store=8448ac25ea038ac8c63c9da8906d8673/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:40,976 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1038): replaying wal for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:40,976 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:40,977 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:40,978 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:40,978 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:40,978 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:40,979 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1048): stopping wal replay for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:40,979 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1060): Cleaning up temporary data for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:40,980 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:40,981 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 45e15b0320018bda387cb884d175aab6; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68389407, jitterRate=0.01908157765865326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:40,982 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1093): writing seq id for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:40,982 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:40,982 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 45e15b0320018bda387cb884d175aab6: Running coprocessor pre-open hook at 1732104820969Writing region info on filesystem at 1732104820969Initializing all the Stores at 1732104820970 (+1 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104820970Cleaning up temporary data from old regions at 1732104820978 (+8 ms)Running coprocessor post-open hooks at 1732104820982 (+4 ms)Region opened successfully at 1732104820982 2024-11-20T12:13:40,983 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for test,sss,1732104815671.45e15b0320018bda387cb884d175aab6., pid=75, masterSystemTime=1732104820924 2024-11-20T12:13:40,983 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1114): Opened 8448ac25ea038ac8c63c9da8906d8673; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62699591, jitterRate=-0.06570328772068024}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:40,984 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:40,984 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegion(1006): Region open journal for 8448ac25ea038ac8c63c9da8906d8673: Running coprocessor pre-open hook at 1732104820971Writing region info on filesystem at 1732104820971Initializing all the Stores at 1732104820973 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104820973Cleaning up temporary data from old regions at 1732104820979 (+6 ms)Running coprocessor post-open hooks at 1732104820984 (+5 ms)Region opened successfully at 1732104820984 2024-11-20T12:13:40,986 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2236): Post open deploy tasks for test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673., pid=81, masterSystemTime=1732104820931 2024-11-20T12:13:40,991 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:40,991 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:40,991 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(132): Open test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:40,992 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7752): Opening region: {ENCODED => abac5b1bb17e965219c5ca01a8ebc325, NAME => 'test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325.', STARTKEY => 'ggg', ENDKEY => 'hhh'} 2024-11-20T12:13:40,992 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:40,992 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(898): Instantiated test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:40,992 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7794): checking encryption for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:40,992 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(7797): checking classloading for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:40,995 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] regionserver.HRegionServer(2266): Finished post open deploy task for test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:40,995 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=81}] handler.AssignRegionHandler(153): Opened test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:40,995 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(132): Open test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:40,995 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7752): Opening region: {ENCODED => 03e3b3cc4355d1bb26b41172ffed46ca, NAME => 'test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca.', STARTKEY => 'ppp', ENDKEY => 'qqq'} 2024-11-20T12:13:40,995 INFO [StoreOpener-abac5b1bb17e965219c5ca01a8ebc325-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:40,996 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table test 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:40,996 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(898): Instantiated test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-20T12:13:40,996 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7794): checking encryption for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:40,996 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(7797): checking classloading for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:40,997 INFO [StoreOpener-abac5b1bb17e965219c5ca01a8ebc325-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region abac5b1bb17e965219c5ca01a8ebc325 columnFamilyName family 2024-11-20T12:13:40,997 DEBUG [StoreOpener-abac5b1bb17e965219c5ca01a8ebc325-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:40,998 INFO [StoreOpener-abac5b1bb17e965219c5ca01a8ebc325-1 {}] regionserver.HStore(327): Store=abac5b1bb17e965219c5ca01a8ebc325/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:40,998 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1038): replaying wal for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:40,998 INFO [StoreOpener-03e3b3cc4355d1bb26b41172ffed46ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family family of region 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:40,999 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:41,000 INFO [StoreOpener-03e3b3cc4355d1bb26b41172ffed46ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 03e3b3cc4355d1bb26b41172ffed46ca columnFamilyName family 2024-11-20T12:13:41,001 DEBUG [StoreOpener-03e3b3cc4355d1bb26b41172ffed46ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-20T12:13:41,002 INFO [StoreOpener-03e3b3cc4355d1bb26b41172ffed46ca-1 {}] regionserver.HStore(327): Store=03e3b3cc4355d1bb26b41172ffed46ca/family, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-20T12:13:41,002 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1038): replaying wal for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:41,002 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:41,003 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1048): stopping wal replay for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:41,003 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1060): Cleaning up temporary data for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:41,003 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:41,005 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:41,006 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1048): stopping wal replay for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:41,006 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1060): Cleaning up temporary data for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:41,006 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1093): writing seq id for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:41,008 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1114): Opened abac5b1bb17e965219c5ca01a8ebc325; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71395914, jitterRate=0.0638820230960846}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:41,008 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1122): Running coprocessor post-open hooks for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:41,008 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1093): writing seq id for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:41,008 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegion(1006): Region open journal for abac5b1bb17e965219c5ca01a8ebc325: Running coprocessor pre-open hook at 1732104820992Writing region info on filesystem at 1732104820992Initializing all the Stores at 1732104820995 (+3 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104820995Cleaning up temporary data from old regions at 1732104821004 (+9 ms)Running coprocessor post-open hooks at 1732104821008 (+4 ms)Region opened successfully at 1732104821008 2024-11-20T12:13:41,009 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1114): Opened 03e3b3cc4355d1bb26b41172ffed46ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62231371, jitterRate=-0.07268030941486359}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-20T12:13:41,009 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:41,009 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegion(1006): Region open journal for 03e3b3cc4355d1bb26b41172ffed46ca: Running coprocessor pre-open hook at 1732104820996Writing region info on filesystem at 1732104820996Initializing all the Stores at 1732104820998 (+2 ms)Instantiating store for column family {NAME => 'family', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732104820998Cleaning up temporary data from old regions at 1732104821006 (+8 ms)Running coprocessor post-open hooks at 1732104821009 (+3 ms)Region opened successfully at 1732104821009 2024-11-20T12:13:41,010 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325., pid=76, masterSystemTime=1732104820924 2024-11-20T12:13:41,010 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2236): Post open deploy tasks for test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca., pid=77, masterSystemTime=1732104820931 2024-11-20T12:13:41,013 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:41,013 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=76}] handler.AssignRegionHandler(153): Opened test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:41,014 DEBUG [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] regionserver.HRegionServer(2266): Finished post open deploy task for test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:41,014 INFO [RS_OPEN_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_OPEN_REGION, pid=77}] handler.AssignRegionHandler(153): Opened test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:41,036 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,036 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,036 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,036 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,038 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,038 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,038 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,038 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,039 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T12:13:41,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,048 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=82, ppid=69, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,049 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,050 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,050 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,050 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,051 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,053 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,060 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,060 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8448ac25ea038ac8c63c9da8906d8673, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,061 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b4c46659583192ce01a652d7688d63d, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,062 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,062 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,062 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,063 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,063 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,063 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,063 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,063 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8448ac25ea038ac8c63c9da8906d8673, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,063 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=82, ppid=69, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0b4c46659583192ce01a652d7688d63d, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,064 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 24 msec 2024-11-20T12:13:41,067 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,067 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,067 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,107 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,130 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,130 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,131 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,131 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,131 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,131 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,132 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,132 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN to state=OFFLINE, location=null, table=test, region=2186bb28c010fe9bd50ae06c405d542b to restore RIT 2024-11-20T12:13:41,132 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN to state=OFFLINE, location=null, table=test, region=f42c0bd7eed4146b0b4502022341fd1d to restore RIT 2024-11-20T12:13:41,132 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN to state=OFFLINE, location=null, table=test, region=abac5b1bb17e965219c5ca01a8ebc325 to restore RIT 2024-11-20T12:13:41,132 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851132 2024-11-20T12:13:41,133 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,133 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,134 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,136 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,136 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8448ac25ea038ac8c63c9da8906d8673, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,136 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,136 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8448ac25ea038ac8c63c9da8906d8673, server=843d83d51ba0,38659,1732104813354 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,165 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,166 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,167 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,168 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,169 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,170 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,171 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,172 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,173 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,174 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,174 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,174 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881174 2024-11-20T12:13:41,175 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941175 2024-11-20T12:13:41,175 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 42 msec 2024-11-20T12:13:41,175 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,175 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,175 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,175 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,175 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,176 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,176 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,176 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,177 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T12:13:41,183 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=81, ppid=70, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,184 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,185 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,185 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,185 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,185 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,185 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,187 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,191 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8448ac25ea038ac8c63c9da8906d8673, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,192 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,193 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,193 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=81, ppid=70, state=RUNNABLE, hasLock=false; OpenRegionProcedure 8448ac25ea038ac8c63c9da8906d8673, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,193 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 16 msec 2024-11-20T12:13:41,195 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,195 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,195 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,203 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,203 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,203 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,204 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,204 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,204 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,204 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,204 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,204 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN to state=OFFLINE, location=null, table=test, region=2186bb28c010fe9bd50ae06c405d542b to restore RIT 2024-11-20T12:13:41,204 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN to state=OFFLINE, location=null, table=test, region=f42c0bd7eed4146b0b4502022341fd1d to restore RIT 2024-11-20T12:13:41,205 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN to state=OFFLINE, location=null, table=test, region=abac5b1bb17e965219c5ca01a8ebc325 to restore RIT 2024-11-20T12:13:41,205 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851205 2024-11-20T12:13:41,205 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,205 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,207 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,208 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,208 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,208 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,208 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,44663,1732104813203 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,218 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,219 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,220 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,221 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,222 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,223 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,224 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,225 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,225 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881225 2024-11-20T12:13:41,225 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941225 2024-11-20T12:13:41,225 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 19 msec 2024-11-20T12:13:41,225 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,225 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,225 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,225 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,225 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,225 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,226 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,226 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,227 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,232 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,233 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,233 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=80, ppid=73, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,234 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,234 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,236 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,240 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,241 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,241 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,241 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,242 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,242 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,242 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=80, ppid=73, state=RUNNABLE, hasLock=false; OpenRegionProcedure f42c0bd7eed4146b0b4502022341fd1d, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,242 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 15 msec 2024-11-20T12:13:41,243 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,244 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,244 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,256 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,257 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,257 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,257 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,257 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,257 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,257 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN to state=OFFLINE, location=null, table=test, region=2186bb28c010fe9bd50ae06c405d542b to restore RIT 2024-11-20T12:13:41,258 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN to state=OFFLINE, location=null, table=test, region=f42c0bd7eed4146b0b4502022341fd1d to restore RIT 2024-11-20T12:13:41,258 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN to state=OFFLINE, location=null, table=test, region=abac5b1bb17e965219c5ca01a8ebc325 to restore RIT 2024-11-20T12:13:41,259 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851259 2024-11-20T12:13:41,259 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,259 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,262 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,263 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,263 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,263 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,271 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,273 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,274 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,275 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,276 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,276 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,276 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881276 2024-11-20T12:13:41,276 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941276 2024-11-20T12:13:41,276 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 16 msec 2024-11-20T12:13:41,276 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,276 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,276 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,277 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,277 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,277 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,277 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,277 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,278 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=79, ppid=67, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,283 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,284 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,284 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,285 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,286 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,289 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,290 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,290 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,290 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,291 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,291 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,291 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=79, ppid=67, state=FAILED, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,291 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-20T12:13:41,297 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,297 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,297 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,309 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,309 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,310 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN to state=OFFLINE, location=null, table=test, region=2186bb28c010fe9bd50ae06c405d542b to restore RIT 2024-11-20T12:13:41,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN to state=OFFLINE, location=null, table=test, region=f42c0bd7eed4146b0b4502022341fd1d to restore RIT 2024-11-20T12:13:41,311 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN to state=OFFLINE, location=null, table=test, region=abac5b1bb17e965219c5ca01a8ebc325 to restore RIT 2024-11-20T12:13:41,312 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851312 2024-11-20T12:13:41,312 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,313 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,317 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,321 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,322 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,322 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,322 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,322 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,322 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,323 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,324 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,325 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,326 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,326 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,326 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881326 2024-11-20T12:13:41,326 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941326 2024-11-20T12:13:41,326 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 13 msec 2024-11-20T12:13:41,326 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,326 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,326 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,326 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,327 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,327 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,327 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,327 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,328 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,332 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=78, ppid=72, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,333 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,335 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,338 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,338 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,339 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,340 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,340 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,340 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,340 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,340 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,340 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,340 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,341 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,341 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,341 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=78, ppid=72, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2186bb28c010fe9bd50ae06c405d542b, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,342 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 13 msec 2024-11-20T12:13:41,344 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,344 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,344 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,351 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,351 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,352 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN to state=OFFLINE, location=null, table=test, region=2186bb28c010fe9bd50ae06c405d542b to restore RIT 2024-11-20T12:13:41,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN to state=OFFLINE, location=null, table=test, region=f42c0bd7eed4146b0b4502022341fd1d to restore RIT 2024-11-20T12:13:41,352 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN to state=OFFLINE, location=null, table=test, region=abac5b1bb17e965219c5ca01a8ebc325 to restore RIT 2024-11-20T12:13:41,356 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851356 2024-11-20T12:13:41,356 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,356 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,358 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,361 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,361 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,361 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,361 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,364 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,365 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,366 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,367 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,368 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,368 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881368 2024-11-20T12:13:41,368 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941368 2024-11-20T12:13:41,368 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-20T12:13:41,368 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,368 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,368 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,368 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,369 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,369 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,369 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,369 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,370 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,373 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,373 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,373 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,373 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,373 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,373 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=77, ppid=68, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,373 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,376 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,378 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,379 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,379 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,380 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,380 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,380 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,380 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=77, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 03e3b3cc4355d1bb26b41172ffed46ca, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,380 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-20T12:13:41,381 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,381 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,381 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,393 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,393 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,393 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,393 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,393 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,393 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,393 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,393 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,393 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN to state=OFFLINE, location=null, table=test, region=2186bb28c010fe9bd50ae06c405d542b to restore RIT 2024-11-20T12:13:41,394 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN to state=OFFLINE, location=null, table=test, region=f42c0bd7eed4146b0b4502022341fd1d to restore RIT 2024-11-20T12:13:41,394 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN to state=OFFLINE, location=null, table=test, region=abac5b1bb17e965219c5ca01a8ebc325 to restore RIT 2024-11-20T12:13:41,394 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851394 2024-11-20T12:13:41,394 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,394 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,396 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,397 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,397 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,397 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,402 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,403 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,404 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,405 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,406 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,406 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,406 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881406 2024-11-20T12:13:41,406 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941406 2024-11-20T12:13:41,406 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-20T12:13:41,406 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,406 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,406 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,406 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,407 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,407 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,407 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,408 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,411 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=76, ppid=74, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,412 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,412 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,413 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,415 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,416 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,416 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,416 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,416 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,416 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,416 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,417 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,417 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,417 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,417 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,417 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,417 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=76, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure abac5b1bb17e965219c5ca01a8ebc325, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,417 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 9 msec 2024-11-20T12:13:41,418 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,418 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,418 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,424 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,424 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,424 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN to state=OFFLINE, location=null, table=test, region=2186bb28c010fe9bd50ae06c405d542b to restore RIT 2024-11-20T12:13:41,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN to state=OFFLINE, location=null, table=test, region=f42c0bd7eed4146b0b4502022341fd1d to restore RIT 2024-11-20T12:13:41,425 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN to state=OFFLINE, location=null, table=test, region=abac5b1bb17e965219c5ca01a8ebc325 to restore RIT 2024-11-20T12:13:41,426 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851426 2024-11-20T12:13:41,426 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,426 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,428 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,429 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,429 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,429 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,433 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,434 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,435 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,438 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,438 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881438 2024-11-20T12:13:41,438 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941438 2024-11-20T12:13:41,438 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-20T12:13:41,438 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,438 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,438 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,438 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,439 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,439 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,439 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,439 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,440 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,443 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=75, ppid=71, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,444 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,444 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,444 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,445 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,445 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,446 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,447 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,449 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,450 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,451 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,451 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,451 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=75, ppid=71, state=RUNNABLE, hasLock=false; OpenRegionProcedure 45e15b0320018bda387cb884d175aab6, server=843d83d51ba0,44663,1732104813203 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,451 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 11 msec 2024-11-20T12:13:41,452 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,452 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,453 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,463 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,464 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,464 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN to state=OFFLINE, location=null, table=test, region=2186bb28c010fe9bd50ae06c405d542b to restore RIT 2024-11-20T12:13:41,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN to state=OFFLINE, location=null, table=test, region=f42c0bd7eed4146b0b4502022341fd1d to restore RIT 2024-11-20T12:13:41,465 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN to state=OFFLINE, location=null, table=test, region=abac5b1bb17e965219c5ca01a8ebc325 to restore RIT 2024-11-20T12:13:41,465 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851465 2024-11-20T12:13:41,465 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,466 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,467 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,468 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,469 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,469 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,472 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,473 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,474 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,475 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,476 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,477 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,477 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,477 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881477 2024-11-20T12:13:41,477 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941477 2024-11-20T12:13:41,477 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 11 msec 2024-11-20T12:13:41,477 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,477 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,477 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,478 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,478 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,478 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,478 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,478 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,479 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,482 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=74, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,483 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,483 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,484 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,485 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,485 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,485 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=74, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,486 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,486 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,486 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,486 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,487 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,487 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,487 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,487 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,487 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,487 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,487 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,487 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,487 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN 2024-11-20T12:13:41,487 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T12:13:41,488 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,488 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,488 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,498 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,498 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,498 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,507 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,507 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,508 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,508 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,508 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,508 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN to state=OFFLINE, location=null, table=test, region=2186bb28c010fe9bd50ae06c405d542b to restore RIT 2024-11-20T12:13:41,508 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN to state=OFFLINE, location=null, table=test, region=f42c0bd7eed4146b0b4502022341fd1d to restore RIT 2024-11-20T12:13:41,508 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=74, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=abac5b1bb17e965219c5ca01a8ebc325, ASSIGN to state=OFFLINE, location=null, table=test, region=abac5b1bb17e965219c5ca01a8ebc325 to restore RIT 2024-11-20T12:13:41,512 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851512 2024-11-20T12:13:41,512 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,512 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,514 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,515 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,515 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,515 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,520 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,521 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,522 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,523 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,524 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,525 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,525 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,525 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881525 2024-11-20T12:13:41,525 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941525 2024-11-20T12:13:41,525 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 12 msec 2024-11-20T12:13:41,525 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,525 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,525 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,525 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,526 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,526 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,526 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,526 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,527 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,529 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=73, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,530 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,530 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,531 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,531 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,532 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=73, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,533 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,533 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,533 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,534 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,534 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN 2024-11-20T12:13:41,534 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T12:13:41,535 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,535 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,536 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,548 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,548 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,549 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,549 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,549 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,549 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,550 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,551 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,551 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,551 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,551 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,551 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN to state=OFFLINE, location=null, table=test, region=2186bb28c010fe9bd50ae06c405d542b to restore RIT 2024-11-20T12:13:41,551 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=73, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=f42c0bd7eed4146b0b4502022341fd1d, ASSIGN to state=OFFLINE, location=null, table=test, region=f42c0bd7eed4146b0b4502022341fd1d to restore RIT 2024-11-20T12:13:41,551 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851551 2024-11-20T12:13:41,551 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,551 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,552 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,553 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,553 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,554 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,554 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,558 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,559 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,560 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,561 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,562 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,562 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,562 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881562 2024-11-20T12:13:41,562 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941562 2024-11-20T12:13:41,562 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-20T12:13:41,562 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,562 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,562 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,562 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,563 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,563 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,563 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,563 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,564 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,566 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,567 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,567 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,567 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=72, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,567 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,567 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,567 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,567 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,567 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,568 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,569 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,569 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=72, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,570 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,570 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,571 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,571 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,571 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,571 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,571 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,571 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,571 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,571 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,571 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN 2024-11-20T12:13:41,571 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T12:13:41,572 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,572 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,572 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,582 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,582 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,583 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,584 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,584 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,584 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,584 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=72, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=2186bb28c010fe9bd50ae06c405d542b, ASSIGN to state=OFFLINE, location=null, table=test, region=2186bb28c010fe9bd50ae06c405d542b to restore RIT 2024-11-20T12:13:41,588 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851588 2024-11-20T12:13:41,588 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,588 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,590 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,591 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,591 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,591 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,591 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,594 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,595 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,596 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,597 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,598 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,598 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,598 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881598 2024-11-20T12:13:41,598 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941598 2024-11-20T12:13:41,598 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-20T12:13:41,598 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,598 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,598 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,598 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,598 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,599 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,599 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,599 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,600 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,602 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=71, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,603 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,603 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,604 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=71, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,606 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,607 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,607 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,607 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,607 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,607 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,607 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,607 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN 2024-11-20T12:13:41,607 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 7 msec 2024-11-20T12:13:41,608 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,608 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,609 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,622 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,624 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,624 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,624 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,624 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,624 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=71, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=45e15b0320018bda387cb884d175aab6, ASSIGN to state=OFFLINE, location=null, table=test, region=45e15b0320018bda387cb884d175aab6 to restore RIT 2024-11-20T12:13:41,625 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851625 2024-11-20T12:13:41,625 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,625 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,627 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,628 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,628 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,628 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,631 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,632 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,633 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,634 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,634 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,634 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881634 2024-11-20T12:13:41,634 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941634 2024-11-20T12:13:41,634 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-20T12:13:41,635 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,635 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,635 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,635 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,635 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,635 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,635 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,635 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,636 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,638 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=70, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,639 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,639 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,640 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=70, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,641 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,642 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,642 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,642 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,642 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,642 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,642 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,642 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,642 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,642 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,642 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,642 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,642 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,642 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN 2024-11-20T12:13:41,642 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T12:13:41,643 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,643 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,644 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,652 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,652 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,656 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,656 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,656 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,656 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,656 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,656 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,657 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,657 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=70, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=8448ac25ea038ac8c63c9da8906d8673, ASSIGN to state=OFFLINE, location=null, table=test, region=8448ac25ea038ac8c63c9da8906d8673 to restore RIT 2024-11-20T12:13:41,657 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851657 2024-11-20T12:13:41,657 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,657 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,658 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,659 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,659 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,659 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,662 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,663 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,664 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,665 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,666 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,666 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,666 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881666 2024-11-20T12:13:41,666 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941666 2024-11-20T12:13:41,666 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-20T12:13:41,666 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,666 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,666 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,666 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,666 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,666 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,667 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,667 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,668 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:41,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=69, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,669 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,669 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,670 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=69, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,671 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,672 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,672 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,672 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,672 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,672 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,672 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN 2024-11-20T12:13:41,672 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-20T12:13:41,673 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,673 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,673 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,684 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,685 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,685 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=69, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=0b4c46659583192ce01a652d7688d63d, ASSIGN to state=OFFLINE, location=null, table=test, region=0b4c46659583192ce01a652d7688d63d to restore RIT 2024-11-20T12:13:41,686 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851686 2024-11-20T12:13:41,686 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:41,686 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,687 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:41,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,691 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,692 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:41,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:41,693 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,693 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:41,693 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104881693 2024-11-20T12:13:41,693 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104941693 2024-11-20T12:13:41,694 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 7 msec 2024-11-20T12:13:41,694 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:41,694 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,694 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,694 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,944 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:41,944 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:41,944 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:41,944 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:41,945 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:41,945 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:41,945 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:41,946 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:41,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-20T12:13:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=68, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:41,951 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:41,952 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:41,953 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:41,953 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:41,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=68, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:41,956 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:41,956 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN 2024-11-20T12:13:41,956 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 8 msec 2024-11-20T12:13:41,957 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:41,957 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:41,957 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:41,966 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,976 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:41,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:41,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,996 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:41,997 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:41,997 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=68, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=03e3b3cc4355d1bb26b41172ffed46ca, ASSIGN to state=OFFLINE, location=null, table=test, region=03e3b3cc4355d1bb26b41172ffed46ca to restore RIT 2024-11-20T12:13:41,999 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104851999 2024-11-20T12:13:41,999 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:42,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,006 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,007 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:41,999 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,008 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:42,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,008 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,009 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,009 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:42,009 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104882009 2024-11-20T12:13:42,010 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104942009 2024-11-20T12:13:42,010 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:42,010 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 10 msec 2024-11-20T12:13:42,010 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:42,010 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,010 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,260 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:42,260 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,260 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:42,260 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:42,260 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:42,260 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:42,261 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:42,261 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:42,262 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:42,264 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:42,264 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:42,264 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:42,264 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=67, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:42,264 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:42,264 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:42,264 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:42,264 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:42,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:42,265 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:42,266 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:42,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:42,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:42,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,267 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=67, ppid=60, state=WAITING:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:42,267 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:42,267 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,268 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:42,268 INFO [Time-limited test {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN 2024-11-20T12:13:42,268 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-20T12:13:42,272 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:42,272 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:42,273 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:42,307 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,312 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,361 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,362 INFO [Time-limited test {}] assignment.AssignmentManager(385): Attach pid=67, ppid=60, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, hasLock=false; TransitRegionStateProcedure table=test, region=99f9c663ffb10623454992ca39cafe6e, ASSIGN to state=OFFLINE, location=null, table=test, region=99f9c663ffb10623454992ca39cafe6e to restore RIT 2024-11-20T12:13:42,362 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104852362 2024-11-20T12:13:42,363 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:42,363 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,364 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:42,365 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:42,365 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,365 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,365 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,368 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,369 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,370 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,371 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:42,371 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104882371 2024-11-20T12:13:42,371 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104942371 2024-11-20T12:13:42,371 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-20T12:13:42,371 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:42,371 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,371 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:42,371 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:42,371 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:42,371 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:42,371 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:42,372 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:42,372 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:42,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:42,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:42,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=66, ppid=65, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:42,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:42,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:42,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:42,374 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:42,374 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:42,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:42,375 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,376 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=66, ppid=65, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,376 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T12:13:42,378 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:42,378 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:42,378 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:42,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,390 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,392 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104852391 2024-11-20T12:13:42,392 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:42,392 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,393 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:42,395 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:42,395 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,395 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,395 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,398 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,399 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,400 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,400 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:42,400 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104882400 2024-11-20T12:13:42,401 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104942400 2024-11-20T12:13:42,401 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-20T12:13:42,401 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:42,401 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,401 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:42,401 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:42,401 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:42,401 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:42,401 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:42,401 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:42,402 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:42,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:42,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=65, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:42,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:42,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:42,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:42,403 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:42,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:42,404 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:42,405 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:42,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:42,406 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:42,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,407 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,407 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:42,407 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,407 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,407 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,407 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,407 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=65, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.1732104814865, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,407 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-20T12:13:42,409 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:42,409 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:42,409 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:42,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,421 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,422 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,431 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104852431 2024-11-20T12:13:42,431 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:42,431 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,435 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:42,436 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,437 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,438 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:42,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,439 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,440 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,440 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:42,440 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104882440 2024-11-20T12:13:42,440 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104942440 2024-11-20T12:13:42,440 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-20T12:13:42,440 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:42,440 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,440 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,686 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-20T12:13:42,687 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-20T12:13:42,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_test 2024-11-20T12:13:42,688 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_test Metrics about Tables on a single HBase RegionServer 2024-11-20T12:13:42,688 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T12:13:42,689 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-20T12:13:42,689 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-20T12:13:42,689 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-20T12:13:42,690 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:42,690 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,690 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:42,690 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:42,691 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:42,691 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:42,691 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:42,691 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:42,692 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:42,693 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:42,693 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:42,693 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:42,693 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:42,693 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=64, ppid=63, class=org.apache.hadoop.hbase.master.assignment.OpenRegionProcedure) stack ids=[] 2024-11-20T12:13:42,693 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:42,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:42,694 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:42,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:42,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:42,695 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,695 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:42,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,695 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=64, ppid=63, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,695 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T12:13:42,696 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:42,696 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:42,696 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:42,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,729 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,731 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104852730 2024-11-20T12:13:42,731 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:42,731 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,731 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:42,733 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:42,733 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:42,733 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,733 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,736 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,737 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,738 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:42,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,739 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,739 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:42,739 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104882739 2024-11-20T12:13:42,739 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104942739 2024-11-20T12:13:42,739 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-20T12:13:42,740 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:42,740 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,740 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:42,740 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:42,740 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:42,740 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:42,740 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:42,740 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:42,741 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:42,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:42,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:42,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:42,743 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=63, ppid=60, class=org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure) stack ids=[] 2024-11-20T12:13:42,743 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:42,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:42,744 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:42,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:42,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,745 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-20T12:13:42,745 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:42,745 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,745 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,745 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=63, ppid=60, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,745 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T12:13:42,746 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:42,746 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:42,746 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:42,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,758 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,758 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,758 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,759 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104852759 2024-11-20T12:13:42,759 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:42,759 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,761 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:42,763 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:42,763 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,763 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,765 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,766 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:42,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,767 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,768 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,768 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:42,768 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104882768 2024-11-20T12:13:42,768 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104942768 2024-11-20T12:13:42,768 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-20T12:13:42,768 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:42,768 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,768 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:42,768 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:42,769 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:42,769 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:42,769 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:42,769 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:42,770 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:42,772 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:42,772 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:42,772 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=62, ppid=61, class=org.apache.hadoop.hbase.master.procedure.SplitWALRemoteProcedure) stack ids=[] 2024-11-20T12:13:42,772 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:42,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:42,773 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:42,773 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=39157 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerReport(MasterRpcServices.java:630) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16716) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:42,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,773 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,773 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:42,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,774 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=62, ppid=61, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,774 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T12:13:42,774 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:42,775 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:42,775 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:42,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,782 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,782 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,782 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,783 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104852783 2024-11-20T12:13:42,784 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:42,784 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,785 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:42,787 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:42,787 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,787 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1593) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,790 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,791 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:42,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,792 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,792 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:42,793 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104882792 2024-11-20T12:13:42,793 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104942793 2024-11-20T12:13:42,793 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 9 msec 2024-11-20T12:13:42,793 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:42,793 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,793 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:42,793 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:42,793 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:42,793 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:42,793 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:42,793 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:42,794 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:42,796 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:42,796 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=61, ppid=60, class=org.apache.hadoop.hbase.master.procedure.SplitWALProcedure) stack ids=[] 2024-11-20T12:13:42,796 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:42,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:42,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:42,797 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:42,798 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,798 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:42,798 DEBUG [Time-limited test {}] procedure2.Procedure(965): pid=61, ppid=60, state=SUCCESS, hasLock=false; SplitWALProcedure 843d83d51ba0%2C35591%2C1732104813307.meta.1732104815320.meta, worker=843d83d51ba0,38659,1732104813354 didn't hold the lock before restarting, skip acquiring lock. 2024-11-20T12:13:42,798 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T12:13:42,799 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:42,799 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:42,799 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:42,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741848_1024 (size=45) 2024-11-20T12:13:42,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741850_1026 (size=45) 2024-11-20T12:13:42,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741851_1027 (size=45) 2024-11-20T12:13:42,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741847_1023 (size=45) 2024-11-20T12:13:42,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,811 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,820 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104852820 2024-11-20T12:13:42,820 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:42,820 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,822 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:42,823 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: false 2024-11-20T12:13:42,823 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1952): TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:42,823 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,824 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$WorkerThread(2184): Worker terminating UNNATURALLY null java.lang.RuntimeException: TESTING: Kill BEFORE store update in rollback: pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.kill(ProcedureExecutor.java:1954) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.cleanupAfterRollbackOneStep(ProcedureExecutor.java:1721) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeUnexpectedRollback(ProcedureExecutor.java:1604) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeRollback(ProcedureExecutor.java:1701) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1460) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,826 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,827 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:42,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,828 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,829 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,829 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:42,829 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104882829 2024-11-20T12:13:42,829 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104942829 2024-11-20T12:13:42,829 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-20T12:13:42,829 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(118): RESTART - Stop 2024-11-20T12:13:42,829 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,829 INFO [Time-limited test {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:42,829 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:42,829 INFO [Time-limited test {}] procedure2.ProcedureTestingUtility(130): RESTART - Start 2024-11-20T12:13:42,829 INFO [Time-limited test {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=1 2024-11-20T12:13:42,830 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 1 core workers (bigger of cpus/4 or 16) with max (burst) worker count=10 2024-11-20T12:13:42,830 INFO [Time-limited test {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-20T12:13:42,831 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 0 msec 2024-11-20T12:13:42,832 DEBUG [Time-limited test {}] store.ProcedureTree(114): Procedure Procedure(pid=60, ppid=-1, class=org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure) stack ids=[] 2024-11-20T12:13:42,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-20T12:13:42,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=test 2024-11-20T12:13:42,833 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(545): Completed pid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=test, region=113951a1eeaf21f60b028a18a0a1a80b, REOPEN/MOVE 2024-11-20T12:13:42,834 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(420): Loading pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true 2024-11-20T12:13:42,834 DEBUG [Time-limited test {}] procedure2.Procedure(985): pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true held the lock before restarting, call acquireLock to restore it. 2024-11-20T12:13:42,834 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-11-20T12:13:42,835 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=1588230740, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=65 2024-11-20T12:13:42,835 DEBUG [Time-limited test {}] assignment.AssignmentManager(349): Loaded hbase:meta state=OPEN, location=843d83d51ba0,38659,1732104813354, table=hbase:meta, region=1588230740 2024-11-20T12:13:42,835 INFO [Time-limited test {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 843d83d51ba0,38659,1732104813354, state=OPEN 2024-11-20T12:13:42,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,845 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-20T12:13:42,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,845 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-20T12:13:42,847 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732104852847 2024-11-20T12:13:42,847 DEBUG [Time-limited test {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-20T12:13:42,847 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=60, state=FAILED:SERVER_CRASH_FINISH, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.RuntimeException: inject code bug at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureUtil.convertToProcedure(ProcedureUtil.java:266) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.ProtoAndProcedure.getProcedure(ProtoAndProcedure.java:43) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.InMemoryProcedureIterator.next(InMemoryProcedureIterator.java:90) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.loadProcedures(ProcedureExecutor.java:539) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$1.load(ProcedureExecutor.java:344) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.store.region.RegionProcedureStore.load(RegionProcedureStore.java:287) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.load(ProcedureExecutor.java:335) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.init(ProcedureExecutor.java:688) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:132) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility.restart(ProcedureTestingUtility.java:100) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT-tests.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.procedure.MasterProcedureTestingUtility.restartMasterProcedureExecutor(MasterProcedureTestingUtility.java:85) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.testFailAndRollback(TestRollbackSCP.java:180) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.RuntimeException: inject code bug at java.lang.Thread.getStackTrace(Thread.java:1619) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.setStackTrace(FutureUtils.java:144) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:166) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureFutureUtil.suspendIfNecessary(ProcedureFutureUtil.java:92) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at --------Future.get--------(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP$AssignmentManagerForTest.persistToMeta(TestRollbackSCP.java:101) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:332) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.RegionRemoteProcedureBase.execute(RegionRemoteProcedureBase.java:60) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-20T12:13:42,848 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor$Testing(164): Toggle KILL before store update in rollback to: true 2024-11-20T12:13:42,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=60, state=ROLLEDBACK, hasLock=false, exception=java.lang.RuntimeException via CODE-BUG: Uncaught runtime exception: pid=79, ppid=67, state=RUNNABLE, hasLock=true; OpenRegionProcedure 99f9c663ffb10623454992ca39cafe6e, server=843d83d51ba0,44663,1732104813203:java.lang.RuntimeException: inject code bug; ServerCrashProcedure 843d83d51ba0,35591,1732104813307, splitWal=true, meta=true exec-time=3.6010 sec 2024-11-20T12:13:42,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=99f9c663ffb10623454992ca39cafe6e, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,852 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23c5a5a22a2d09813d1ff472bb76f407, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=ae055eeceea830ff461b6b4dc158461e, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=9b2ba9c7430b8a524b3966df315ad45c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8448ac25ea038ac8c63c9da8906d8673, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=325e5f934bae06dfb2390407a9e9726f, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=abac5b1bb17e965219c5ca01a8ebc325, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=6171c0ffde5383e6eb4635a4c76e3ad1, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=dec6f7c7f05d870bae6d81029b3676e2, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0b4c46659583192ce01a652d7688d63d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,853 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c7c42370d5f1944a9469b506921ac97c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=633a6ec91d322a9170f4899c41489180, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=f42c0bd7eed4146b0b4502022341fd1d, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=0a3d655f8e46d7b0303a071be25a13da, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8deafbcc710170143f5159662c491d64, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=03e3b3cc4355d1bb26b41172ffed46ca, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=fef27752c5c42f2f9602a38de6f56349, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=23f5434f0840c36b9d9bb2e81c743305, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=45e15b0320018bda387cb884d175aab6, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,854 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=312c8b2fa94f26f46326bffa7ae2bcff, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=113951a1eeaf21f60b028a18a0a1a80b, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=5 2024-11-20T12:13:42,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=c2a739d456605f6e8ff10b7be87ca995, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=5f89ec3b6d6dbb62bc1c5a7ca523c08c, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=2186bb28c010fe9bd50ae06c405d542b, regionState=OPENING, lastHost=843d83d51ba0,35591,1732104813307, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=8978fd254a3dd3cd4570b36a1c18b8cb, regionState=OPEN, lastHost=843d83d51ba0,38659,1732104813354, regionLocation=843d83d51ba0,38659,1732104813354, openSeqNum=2 2024-11-20T12:13:42,855 DEBUG [Time-limited test {}] assignment.RegionStateStore(171): Load hbase:meta entry region=366d4b256c89d7860c4d7747ae8d7cc6, regionState=OPEN, lastHost=843d83d51ba0,44663,1732104813203, regionLocation=843d83d51ba0,44663,1732104813203, openSeqNum=2 2024-11-20T12:13:42,855 INFO [Time-limited test {}] assignment.AssignmentManager(1768): Number of RegionServers=2 2024-11-20T12:13:42,855 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732104882855 2024-11-20T12:13:42,855 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732104942855 2024-11-20T12:13:42,855 INFO [Time-limited test {}] assignment.AssignmentManager(1775): Joined the cluster in 8 msec 2024-11-20T12:13:42,872 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestRollbackSCP#testFailAndRollback Thread=350 (was 377), OpenFileDescriptor=604 (was 623), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=598 (was 598), ProcessCount=11 (was 11), AvailableMemoryMB=1593 (was 1676) 2024-11-20T12:13:42,873 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-20T12:13:42,873 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T12:13:42,873 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:13:42,873 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:13:42,874 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:13:42,874 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-20T12:13:42,874 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-20T12:13:42,874 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1027511582, stopped=false 2024-11-20T12:13:42,875 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=843d83d51ba0,39157,1732104812301 2024-11-20T12:13:42,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T12:13:42,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T12:13:42,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-20T12:13:42,887 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:42,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:42,887 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:42,888 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-20T12:13:42,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:42,888 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.master.assignment.TestRollbackSCP.tearDownAfterClass(TestRollbackSCP.java:135) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:13:42,888 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:13:42,888 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '843d83d51ba0,44663,1732104813203' ***** 2024-11-20T12:13:42,888 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T12:13:42,888 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T12:13:42,888 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '843d83d51ba0,38659,1732104813354' ***** 2024-11-20T12:13:42,888 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-20T12:13:42,888 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T12:13:42,889 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-20T12:13:42,889 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T12:13:42,889 INFO [RS:0;843d83d51ba0:44663 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T12:13:42,889 INFO [RS:0;843d83d51ba0:44663 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T12:13:42,889 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for c7c42370d5f1944a9469b506921ac97c 2024-11-20T12:13:42,889 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-20T12:13:42,889 INFO [RS:2;843d83d51ba0:38659 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-20T12:13:42,890 INFO [RS:2;843d83d51ba0:38659 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-20T12:13:42,890 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 325e5f934bae06dfb2390407a9e9726f 2024-11-20T12:13:42,890 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for 8deafbcc710170143f5159662c491d64 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for 6171c0ffde5383e6eb4635a4c76e3ad1 2024-11-20T12:13:42,890 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for 366d4b256c89d7860c4d7747ae8d7cc6 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for 23f5434f0840c36b9d9bb2e81c743305 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for 633a6ec91d322a9170f4899c41489180 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for 9b2ba9c7430b8a524b3966df315ad45c 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for 5f89ec3b6d6dbb62bc1c5a7ca523c08c 2024-11-20T12:13:42,890 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 0a3d655f8e46d7b0303a071be25a13da 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for abac5b1bb17e965219c5ca01a8ebc325 2024-11-20T12:13:42,890 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for dec6f7c7f05d870bae6d81029b3676e2 2024-11-20T12:13:42,890 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 03e3b3cc4355d1bb26b41172ffed46ca 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for ae055eeceea830ff461b6b4dc158461e 2024-11-20T12:13:42,890 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 0b4c46659583192ce01a652d7688d63d 2024-11-20T12:13:42,890 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 23c5a5a22a2d09813d1ff472bb76f407 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for 99f9c663ffb10623454992ca39cafe6e 2024-11-20T12:13:42,890 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 8448ac25ea038ac8c63c9da8906d8673 2024-11-20T12:13:42,890 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(3091): Received CLOSE for 45e15b0320018bda387cb884d175aab6 2024-11-20T12:13:42,890 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 312c8b2fa94f26f46326bffa7ae2bcff 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:42,890 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c7c42370d5f1944a9469b506921ac97c, disabling compactions & flushes 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 2186bb28c010fe9bd50ae06c405d542b 2024-11-20T12:13:42,891 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(959): stopping server 843d83d51ba0,44663,1732104813203 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 8978fd254a3dd3cd4570b36a1c18b8cb 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 113951a1eeaf21f60b028a18a0a1a80b 2024-11-20T12:13:42,891 INFO [RS:0;843d83d51ba0:44663 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T12:13:42,891 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for c2a739d456605f6e8ff10b7be87ca995 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(959): stopping server 843d83d51ba0,38659,1732104813354 2024-11-20T12:13:42,891 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:42,891 INFO [RS:0;843d83d51ba0:44663 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;843d83d51ba0:44663. 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;843d83d51ba0:38659. 2024-11-20T12:13:42,891 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. after waiting 0 ms 2024-11-20T12:13:42,891 DEBUG [RS:0;843d83d51ba0:44663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:13:42,891 DEBUG [RS:0;843d83d51ba0:44663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:13:42,891 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:42,891 DEBUG [RS:2;843d83d51ba0:38659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-20T12:13:42,891 DEBUG [RS:2;843d83d51ba0:38659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:13:42,891 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(1321): Waiting on 13 regions to close 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T12:13:42,891 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-20T12:13:42,892 DEBUG [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(1351): Waiting on 23f5434f0840c36b9d9bb2e81c743305, 366d4b256c89d7860c4d7747ae8d7cc6, 45e15b0320018bda387cb884d175aab6, 5f89ec3b6d6dbb62bc1c5a7ca523c08c, 6171c0ffde5383e6eb4635a4c76e3ad1, 633a6ec91d322a9170f4899c41489180, 8deafbcc710170143f5159662c491d64, 99f9c663ffb10623454992ca39cafe6e, 9b2ba9c7430b8a524b3966df315ad45c, abac5b1bb17e965219c5ca01a8ebc325, ae055eeceea830ff461b6b4dc158461e, c7c42370d5f1944a9469b506921ac97c, f42c0bd7eed4146b0b4502022341fd1d 2024-11-20T12:13:42,892 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 325e5f934bae06dfb2390407a9e9726f, disabling compactions & flushes 2024-11-20T12:13:42,892 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:42,892 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:42,892 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. after waiting 0 ms 2024-11-20T12:13:42,892 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:42,892 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(1321): Waiting on 14 regions to close 2024-11-20T12:13:42,893 DEBUG [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(1351): Waiting on 03e3b3cc4355d1bb26b41172ffed46ca, 0a3d655f8e46d7b0303a071be25a13da, 0b4c46659583192ce01a652d7688d63d, 113951a1eeaf21f60b028a18a0a1a80b, 1588230740, 2186bb28c010fe9bd50ae06c405d542b, 23c5a5a22a2d09813d1ff472bb76f407, 312c8b2fa94f26f46326bffa7ae2bcff, 325e5f934bae06dfb2390407a9e9726f, 8448ac25ea038ac8c63c9da8906d8673, 8978fd254a3dd3cd4570b36a1c18b8cb, c2a739d456605f6e8ff10b7be87ca995, dec6f7c7f05d870bae6d81029b3676e2, fef27752c5c42f2f9602a38de6f56349 2024-11-20T12:13:42,893 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-20T12:13:42,893 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-20T12:13:42,893 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-20T12:13:42,893 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-20T12:13:42,893 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-20T12:13:42,893 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=2.64 KB heapSize=5.42 KB 2024-11-20T12:13:42,900 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/c7c42370d5f1944a9469b506921ac97c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:42,902 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:42,902 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c7c42370d5f1944a9469b506921ac97c: Waiting for close lock at 1732104822890Running coprocessor pre-close hooks at 1732104822890Disabling compacts and flushes for region at 1732104822890Disabling writes for close at 1732104822891 (+1 ms)Writing region close event to WAL at 1732104822892 (+1 ms)Running coprocessor post-close hooks at 1732104822902 (+10 ms)Closed at 1732104822902 2024-11-20T12:13:42,902 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,kkk,1732104815671.c7c42370d5f1944a9469b506921ac97c. 2024-11-20T12:13:42,902 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8deafbcc710170143f5159662c491d64, disabling compactions & flushes 2024-11-20T12:13:42,903 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:42,903 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:42,903 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. after waiting 0 ms 2024-11-20T12:13:42,903 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:42,904 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/325e5f934bae06dfb2390407a9e9726f/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:42,905 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:42,905 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 325e5f934bae06dfb2390407a9e9726f: Waiting for close lock at 1732104822892Running coprocessor pre-close hooks at 1732104822892Disabling compacts and flushes for region at 1732104822892Disabling writes for close at 1732104822892Writing region close event to WAL at 1732104822898 (+6 ms)Running coprocessor post-close hooks at 1732104822905 (+7 ms)Closed at 1732104822905 2024-11-20T12:13:42,906 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,fff,1732104815671.325e5f934bae06dfb2390407a9e9726f. 2024-11-20T12:13:42,906 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0a3d655f8e46d7b0303a071be25a13da, disabling compactions & flushes 2024-11-20T12:13:42,906 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:42,906 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:42,906 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. after waiting 0 ms 2024-11-20T12:13:42,906 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:42,907 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/info/dd4fdb017e364ce28bb975439450befb is 138, key is test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673./info:regioninfo/1732104820659/Put/seqid=0 2024-11-20T12:13:42,908 INFO [regionserver/843d83d51ba0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T12:13:42,913 INFO [regionserver/843d83d51ba0:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T12:13:42,921 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8deafbcc710170143f5159662c491d64/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:42,921 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/0a3d655f8e46d7b0303a071be25a13da/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:42,922 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:42,922 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8deafbcc710170143f5159662c491d64: Waiting for close lock at 1732104822902Running coprocessor pre-close hooks at 1732104822902Disabling compacts and flushes for region at 1732104822902Disabling writes for close at 1732104822903 (+1 ms)Writing region close event to WAL at 1732104822904 (+1 ms)Running coprocessor post-close hooks at 1732104822922 (+18 ms)Closed at 1732104822922 2024-11-20T12:13:42,922 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:42,922 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ooo,1732104815671.8deafbcc710170143f5159662c491d64. 2024-11-20T12:13:42,922 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0a3d655f8e46d7b0303a071be25a13da: Waiting for close lock at 1732104822906Running coprocessor pre-close hooks at 1732104822906Disabling compacts and flushes for region at 1732104822906Disabling writes for close at 1732104822906Writing region close event to WAL at 1732104822916 (+10 ms)Running coprocessor post-close hooks at 1732104822922 (+6 ms)Closed at 1732104822922 2024-11-20T12:13:42,923 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,nnn,1732104815671.0a3d655f8e46d7b0303a071be25a13da. 2024-11-20T12:13:42,923 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 6171c0ffde5383e6eb4635a4c76e3ad1, disabling compactions & flushes 2024-11-20T12:13:42,923 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:42,923 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:42,923 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing dec6f7c7f05d870bae6d81029b3676e2, disabling compactions & flushes 2024-11-20T12:13:42,923 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. after waiting 0 ms 2024-11-20T12:13:42,923 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:42,923 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:42,923 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:42,923 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. after waiting 0 ms 2024-11-20T12:13:42,923 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:42,938 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/6171c0ffde5383e6eb4635a4c76e3ad1/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:42,939 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/dec6f7c7f05d870bae6d81029b3676e2/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:42,940 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 6171c0ffde5383e6eb4635a4c76e3ad1: Waiting for close lock at 1732104822923Running coprocessor pre-close hooks at 1732104822923Disabling compacts and flushes for region at 1732104822923Disabling writes for close at 1732104822923Writing region close event to WAL at 1732104822924 (+1 ms)Running coprocessor post-close hooks at 1732104822939 (+15 ms)Closed at 1732104822939 2024-11-20T12:13:42,940 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for dec6f7c7f05d870bae6d81029b3676e2: Waiting for close lock at 1732104822923Running coprocessor pre-close hooks at 1732104822923Disabling compacts and flushes for region at 1732104822923Disabling writes for close at 1732104822923Writing region close event to WAL at 1732104822928 (+5 ms)Running coprocessor post-close hooks at 1732104822940 (+12 ms)Closed at 1732104822940 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,hhh,1732104815671.6171c0ffde5383e6eb4635a4c76e3ad1. 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,iii,1732104815671.dec6f7c7f05d870bae6d81029b3676e2. 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 366d4b256c89d7860c4d7747ae8d7cc6, disabling compactions & flushes 2024-11-20T12:13:42,940 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 03e3b3cc4355d1bb26b41172ffed46ca, disabling compactions & flushes 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:42,940 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. after waiting 0 ms 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. after waiting 0 ms 2024-11-20T12:13:42,940 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:42,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741869_1045 (size=8177) 2024-11-20T12:13:42,944 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2.64 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/info/dd4fdb017e364ce28bb975439450befb 2024-11-20T12:13:42,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741869_1045 (size=8177) 2024-11-20T12:13:42,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741869_1045 (size=8177) 2024-11-20T12:13:42,955 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/.tmp/info/dd4fdb017e364ce28bb975439450befb as hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/info/dd4fdb017e364ce28bb975439450befb 2024-11-20T12:13:42,967 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/info/dd4fdb017e364ce28bb975439450befb, entries=24, sequenceid=75, filesize=8.0 K 2024-11-20T12:13:42,969 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=75, compaction requested=false 2024-11-20T12:13:42,969 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/366d4b256c89d7860c4d7747ae8d7cc6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:42,970 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/03e3b3cc4355d1bb26b41172ffed46ca/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:42,970 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:42,970 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 366d4b256c89d7860c4d7747ae8d7cc6: Waiting for close lock at 1732104822940Running coprocessor pre-close hooks at 1732104822940Disabling compacts and flushes for region at 1732104822940Disabling writes for close at 1732104822940Writing region close event to WAL at 1732104822941 (+1 ms)Running coprocessor post-close hooks at 1732104822970 (+29 ms)Closed at 1732104822970 2024-11-20T12:13:42,970 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,zzz,1732104815671.366d4b256c89d7860c4d7747ae8d7cc6. 2024-11-20T12:13:42,971 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 23f5434f0840c36b9d9bb2e81c743305, disabling compactions & flushes 2024-11-20T12:13:42,971 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:42,971 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:42,971 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. after waiting 0 ms 2024-11-20T12:13:42,971 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:42,971 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:42,971 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 03e3b3cc4355d1bb26b41172ffed46ca: Waiting for close lock at 1732104822940Running coprocessor pre-close hooks at 1732104822940Disabling compacts and flushes for region at 1732104822940Disabling writes for close at 1732104822940Writing region close event to WAL at 1732104822957 (+17 ms)Running coprocessor post-close hooks at 1732104822971 (+14 ms)Closed at 1732104822971 2024-11-20T12:13:42,971 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ppp,1732104815671.03e3b3cc4355d1bb26b41172ffed46ca. 2024-11-20T12:13:42,971 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 0b4c46659583192ce01a652d7688d63d, disabling compactions & flushes 2024-11-20T12:13:42,971 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:42,971 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:42,972 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. after waiting 0 ms 2024-11-20T12:13:42,972 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:42,984 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/23f5434f0840c36b9d9bb2e81c743305/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:42,985 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:42,985 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 23f5434f0840c36b9d9bb2e81c743305: Waiting for close lock at 1732104822971Running coprocessor pre-close hooks at 1732104822971Disabling compacts and flushes for region at 1732104822971Disabling writes for close at 1732104822971Writing region close event to WAL at 1732104822980 (+9 ms)Running coprocessor post-close hooks at 1732104822985 (+5 ms)Closed at 1732104822985 2024-11-20T12:13:42,985 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,rrr,1732104815671.23f5434f0840c36b9d9bb2e81c743305. 2024-11-20T12:13:42,986 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 633a6ec91d322a9170f4899c41489180, disabling compactions & flushes 2024-11-20T12:13:42,986 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:42,986 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:42,986 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. after waiting 0 ms 2024-11-20T12:13:42,986 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:42,989 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/0b4c46659583192ce01a652d7688d63d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:42,990 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:42,990 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 0b4c46659583192ce01a652d7688d63d: Waiting for close lock at 1732104822971Running coprocessor pre-close hooks at 1732104822971Disabling compacts and flushes for region at 1732104822971Disabling writes for close at 1732104822972 (+1 ms)Writing region close event to WAL at 1732104822976 (+4 ms)Running coprocessor post-close hooks at 1732104822990 (+14 ms)Closed at 1732104822990 2024-11-20T12:13:42,991 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,jjj,1732104815671.0b4c46659583192ce01a652d7688d63d. 2024-11-20T12:13:42,991 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 23c5a5a22a2d09813d1ff472bb76f407, disabling compactions & flushes 2024-11-20T12:13:42,991 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:42,991 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:42,991 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. after waiting 0 ms 2024-11-20T12:13:42,991 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:43,005 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/633a6ec91d322a9170f4899c41489180/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,005 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/hbase/meta/1588230740/recovered.edits/78.seqid, newMaxSeqId=78, maxSeqId=64 2024-11-20T12:13:43,005 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/23c5a5a22a2d09813d1ff472bb76f407/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,006 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:43,006 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 633a6ec91d322a9170f4899c41489180: Waiting for close lock at 1732104822986Running coprocessor pre-close hooks at 1732104822986Disabling compacts and flushes for region at 1732104822986Disabling writes for close at 1732104822986Writing region close event to WAL at 1732104822992 (+6 ms)Running coprocessor post-close hooks at 1732104823006 (+14 ms)Closed at 1732104823006 2024-11-20T12:13:43,006 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 23c5a5a22a2d09813d1ff472bb76f407: Waiting for close lock at 1732104822991Running coprocessor pre-close hooks at 1732104822991Disabling compacts and flushes for region at 1732104822991Disabling writes for close at 1732104822991Writing region close event to WAL at 1732104822993 (+2 ms)Running coprocessor post-close hooks at 1732104823006 (+13 ms)Closed at 1732104823006 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,lll,1732104815671.633a6ec91d322a9170f4899c41489180. 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,bbb,1732104815671.23c5a5a22a2d09813d1ff472bb76f407. 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 9b2ba9c7430b8a524b3966df315ad45c, disabling compactions & flushes 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-20T12:13:43,007 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8448ac25ea038ac8c63c9da8906d8673, disabling compactions & flushes 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:43,007 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. after waiting 0 ms 2024-11-20T12:13:43,007 INFO [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. after waiting 0 ms 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732104822893Running coprocessor pre-close hooks at 1732104822893Disabling compacts and flushes for region at 1732104822893Disabling writes for close at 1732104822893Obtaining lock to block concurrent updates at 1732104822893Preparing flush snapshotting stores in 1588230740 at 1732104822893Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=2708, getHeapSize=5488, getOffHeapSize=0, getCellsCount=24 at 1732104822899 (+6 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732104822901 (+2 ms)Flushing 1588230740/info: creating writer at 1732104822901Flushing 1588230740/info: appending metadata at 1732104822906 (+5 ms)Flushing 1588230740/info: closing flushed file at 1732104822906Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@225bec17: reopening flushed file at 1732104822954 (+48 ms)Finished flush of dataSize ~2.64 KB/2708, heapSize ~4.66 KB/4768, currentSize=0 B/0 for 1588230740 in 75ms, sequenceid=75, compaction requested=false at 1732104822969 (+15 ms)Writing region close event to WAL at 1732104822992 (+23 ms)Running coprocessor post-close hooks at 1732104823006 (+14 ms)Closed at 1732104823007 (+1 ms) 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:43,007 DEBUG [RS_CLOSE_META-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-20T12:13:43,021 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/9b2ba9c7430b8a524b3966df315ad45c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,022 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:43,022 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 9b2ba9c7430b8a524b3966df315ad45c: Waiting for close lock at 1732104823007Running coprocessor pre-close hooks at 1732104823007Disabling compacts and flushes for region at 1732104823007Disabling writes for close at 1732104823007Writing region close event to WAL at 1732104823016 (+9 ms)Running coprocessor post-close hooks at 1732104823022 (+6 ms)Closed at 1732104823022 2024-11-20T12:13:43,022 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ddd,1732104815671.9b2ba9c7430b8a524b3966df315ad45c. 2024-11-20T12:13:43,022 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing f42c0bd7eed4146b0b4502022341fd1d, disabling compactions & flushes 2024-11-20T12:13:43,022 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:43,022 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:43,022 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. after waiting 0 ms 2024-11-20T12:13:43,022 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:43,031 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8448ac25ea038ac8c63c9da8906d8673/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,032 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:43,032 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8448ac25ea038ac8c63c9da8906d8673: Waiting for close lock at 1732104823007Running coprocessor pre-close hooks at 1732104823007Disabling compacts and flushes for region at 1732104823007Disabling writes for close at 1732104823007Writing region close event to WAL at 1732104823017 (+10 ms)Running coprocessor post-close hooks at 1732104823032 (+15 ms)Closed at 1732104823032 2024-11-20T12:13:43,032 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,eee,1732104815671.8448ac25ea038ac8c63c9da8906d8673. 2024-11-20T12:13:43,032 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 312c8b2fa94f26f46326bffa7ae2bcff, disabling compactions & flushes 2024-11-20T12:13:43,032 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:43,032 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:43,033 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. after waiting 0 ms 2024-11-20T12:13:43,033 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:43,035 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/f42c0bd7eed4146b0b4502022341fd1d/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,036 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:43,036 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for f42c0bd7eed4146b0b4502022341fd1d: Waiting for close lock at 1732104823022Running coprocessor pre-close hooks at 1732104823022Disabling compacts and flushes for region at 1732104823022Disabling writes for close at 1732104823022Writing region close event to WAL at 1732104823030 (+8 ms)Running coprocessor post-close hooks at 1732104823035 (+5 ms)Closed at 1732104823036 (+1 ms) 2024-11-20T12:13:43,036 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,mmm,1732104815671.f42c0bd7eed4146b0b4502022341fd1d. 2024-11-20T12:13:43,036 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 5f89ec3b6d6dbb62bc1c5a7ca523c08c, disabling compactions & flushes 2024-11-20T12:13:43,036 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:43,036 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:43,036 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. after waiting 0 ms 2024-11-20T12:13:43,036 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:43,038 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/312c8b2fa94f26f46326bffa7ae2bcff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,038 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:43,038 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 312c8b2fa94f26f46326bffa7ae2bcff: Waiting for close lock at 1732104823032Running coprocessor pre-close hooks at 1732104823032Disabling compacts and flushes for region at 1732104823032Disabling writes for close at 1732104823033 (+1 ms)Writing region close event to WAL at 1732104823033Running coprocessor post-close hooks at 1732104823038 (+5 ms)Closed at 1732104823038 2024-11-20T12:13:43,039 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ttt,1732104815671.312c8b2fa94f26f46326bffa7ae2bcff. 2024-11-20T12:13:43,039 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fef27752c5c42f2f9602a38de6f56349, disabling compactions & flushes 2024-11-20T12:13:43,039 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:43,039 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:43,039 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. after waiting 0 ms 2024-11-20T12:13:43,039 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:43,041 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/5f89ec3b6d6dbb62bc1c5a7ca523c08c/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,042 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:43,042 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 5f89ec3b6d6dbb62bc1c5a7ca523c08c: Waiting for close lock at 1732104823036Running coprocessor pre-close hooks at 1732104823036Disabling compacts and flushes for region at 1732104823036Disabling writes for close at 1732104823036Writing region close event to WAL at 1732104823037 (+1 ms)Running coprocessor post-close hooks at 1732104823042 (+5 ms)Closed at 1732104823042 2024-11-20T12:13:43,042 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,www,1732104815671.5f89ec3b6d6dbb62bc1c5a7ca523c08c. 2024-11-20T12:13:43,042 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing abac5b1bb17e965219c5ca01a8ebc325, disabling compactions & flushes 2024-11-20T12:13:43,042 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:43,043 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:43,043 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. after waiting 0 ms 2024-11-20T12:13:43,043 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:43,049 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/fef27752c5c42f2f9602a38de6f56349/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,049 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/abac5b1bb17e965219c5ca01a8ebc325/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,050 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:43,050 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fef27752c5c42f2f9602a38de6f56349: Waiting for close lock at 1732104823039Running coprocessor pre-close hooks at 1732104823039Disabling compacts and flushes for region at 1732104823039Disabling writes for close at 1732104823039Writing region close event to WAL at 1732104823040 (+1 ms)Running coprocessor post-close hooks at 1732104823050 (+10 ms)Closed at 1732104823050 2024-11-20T12:13:43,050 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:43,050 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,qqq,1732104815671.fef27752c5c42f2f9602a38de6f56349. 2024-11-20T12:13:43,050 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for abac5b1bb17e965219c5ca01a8ebc325: Waiting for close lock at 1732104823042Running coprocessor pre-close hooks at 1732104823042Disabling compacts and flushes for region at 1732104823042Disabling writes for close at 1732104823043 (+1 ms)Writing region close event to WAL at 1732104823044 (+1 ms)Running coprocessor post-close hooks at 1732104823050 (+6 ms)Closed at 1732104823050 2024-11-20T12:13:43,050 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ggg,1732104815671.abac5b1bb17e965219c5ca01a8ebc325. 2024-11-20T12:13:43,050 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 2186bb28c010fe9bd50ae06c405d542b, disabling compactions & flushes 2024-11-20T12:13:43,050 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:43,050 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:43,050 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. after waiting 0 ms 2024-11-20T12:13:43,050 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:43,050 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing ae055eeceea830ff461b6b4dc158461e, disabling compactions & flushes 2024-11-20T12:13:43,050 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:43,050 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:43,050 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. after waiting 0 ms 2024-11-20T12:13:43,051 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:43,056 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/2186bb28c010fe9bd50ae06c405d542b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,058 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:43,058 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 2186bb28c010fe9bd50ae06c405d542b: Waiting for close lock at 1732104823050Running coprocessor pre-close hooks at 1732104823050Disabling compacts and flushes for region at 1732104823050Disabling writes for close at 1732104823050Writing region close event to WAL at 1732104823051 (+1 ms)Running coprocessor post-close hooks at 1732104823058 (+7 ms)Closed at 1732104823058 2024-11-20T12:13:43,058 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,xxx,1732104815671.2186bb28c010fe9bd50ae06c405d542b. 2024-11-20T12:13:43,059 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 8978fd254a3dd3cd4570b36a1c18b8cb, disabling compactions & flushes 2024-11-20T12:13:43,059 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:43,059 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:43,059 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. after waiting 0 ms 2024-11-20T12:13:43,059 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:43,061 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/ae055eeceea830ff461b6b4dc158461e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,062 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:43,062 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for ae055eeceea830ff461b6b4dc158461e: Waiting for close lock at 1732104823050Running coprocessor pre-close hooks at 1732104823050Disabling compacts and flushes for region at 1732104823050Disabling writes for close at 1732104823051 (+1 ms)Writing region close event to WAL at 1732104823052 (+1 ms)Running coprocessor post-close hooks at 1732104823062 (+10 ms)Closed at 1732104823062 2024-11-20T12:13:43,062 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,ccc,1732104815671.ae055eeceea830ff461b6b4dc158461e. 2024-11-20T12:13:43,062 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 99f9c663ffb10623454992ca39cafe6e, disabling compactions & flushes 2024-11-20T12:13:43,063 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:43,063 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:43,063 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. after waiting 0 ms 2024-11-20T12:13:43,063 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:43,065 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/8978fd254a3dd3cd4570b36a1c18b8cb/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,066 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:43,066 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 8978fd254a3dd3cd4570b36a1c18b8cb: Waiting for close lock at 1732104823059Running coprocessor pre-close hooks at 1732104823059Disabling compacts and flushes for region at 1732104823059Disabling writes for close at 1732104823059Writing region close event to WAL at 1732104823060 (+1 ms)Running coprocessor post-close hooks at 1732104823065 (+5 ms)Closed at 1732104823065 2024-11-20T12:13:43,066 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,yyy,1732104815671.8978fd254a3dd3cd4570b36a1c18b8cb. 2024-11-20T12:13:43,066 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 113951a1eeaf21f60b028a18a0a1a80b, disabling compactions & flushes 2024-11-20T12:13:43,066 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:43,066 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:43,066 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. after waiting 0 ms 2024-11-20T12:13:43,066 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:43,072 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/99f9c663ffb10623454992ca39cafe6e/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,072 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/113951a1eeaf21f60b028a18a0a1a80b/recovered.edits/7.seqid, newMaxSeqId=7, maxSeqId=4 2024-11-20T12:13:43,073 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:43,073 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 99f9c663ffb10623454992ca39cafe6e: Waiting for close lock at 1732104823062Running coprocessor pre-close hooks at 1732104823062Disabling compacts and flushes for region at 1732104823062Disabling writes for close at 1732104823063 (+1 ms)Writing region close event to WAL at 1732104823063Running coprocessor post-close hooks at 1732104823073 (+10 ms)Closed at 1732104823073 2024-11-20T12:13:43,073 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:43,073 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 113951a1eeaf21f60b028a18a0a1a80b: Waiting for close lock at 1732104823066Running coprocessor pre-close hooks at 1732104823066Disabling compacts and flushes for region at 1732104823066Disabling writes for close at 1732104823066Writing region close event to WAL at 1732104823067 (+1 ms)Running coprocessor post-close hooks at 1732104823073 (+6 ms)Closed at 1732104823073 2024-11-20T12:13:43,073 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,,1732104815671.99f9c663ffb10623454992ca39cafe6e. 2024-11-20T12:13:43,073 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,uuu,1732104815671.113951a1eeaf21f60b028a18a0a1a80b. 2024-11-20T12:13:43,073 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 45e15b0320018bda387cb884d175aab6, disabling compactions & flushes 2024-11-20T12:13:43,073 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing c2a739d456605f6e8ff10b7be87ca995, disabling compactions & flushes 2024-11-20T12:13:43,073 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:43,073 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:43,073 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:43,074 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:43,074 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. after waiting 0 ms 2024-11-20T12:13:43,074 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. after waiting 0 ms 2024-11-20T12:13:43,074 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:43,074 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:43,079 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/c2a739d456605f6e8ff10b7be87ca995/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,079 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/data/default/test/45e15b0320018bda387cb884d175aab6/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-20T12:13:43,080 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:43,080 INFO [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:43,080 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for c2a739d456605f6e8ff10b7be87ca995: Waiting for close lock at 1732104823073Running coprocessor pre-close hooks at 1732104823073Disabling compacts and flushes for region at 1732104823073Disabling writes for close at 1732104823074 (+1 ms)Writing region close event to WAL at 1732104823074Running coprocessor post-close hooks at 1732104823080 (+6 ms)Closed at 1732104823080 2024-11-20T12:13:43,080 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 45e15b0320018bda387cb884d175aab6: Waiting for close lock at 1732104823073Running coprocessor pre-close hooks at 1732104823073Disabling compacts and flushes for region at 1732104823073Disabling writes for close at 1732104823074 (+1 ms)Writing region close event to WAL at 1732104823074Running coprocessor post-close hooks at 1732104823080 (+6 ms)Closed at 1732104823080 2024-11-20T12:13:43,080 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,vvv,1732104815671.c2a739d456605f6e8ff10b7be87ca995. 2024-11-20T12:13:43,080 DEBUG [RS_CLOSE_REGION-regionserver/843d83d51ba0:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed test,sss,1732104815671.45e15b0320018bda387cb884d175aab6. 2024-11-20T12:13:43,092 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(976): stopping server 843d83d51ba0,44663,1732104813203; all regions closed. 2024-11-20T12:13:43,093 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.HRegionServer(976): stopping server 843d83d51ba0,38659,1732104813354; all regions closed. 2024-11-20T12:13:43,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741865_1041 (size=4192) 2024-11-20T12:13:43,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741835_1011 (size=7064) 2024-11-20T12:13:43,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741835_1011 (size=7064) 2024-11-20T12:13:43,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741835_1011 (size=7064) 2024-11-20T12:13:43,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741865_1041 (size=4192) 2024-11-20T12:13:43,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741865_1041 (size=4192) 2024-11-20T12:13:43,105 DEBUG [RS:2;843d83d51ba0:38659 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/oldWALs 2024-11-20T12:13:43,105 INFO [RS:2;843d83d51ba0:38659 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 843d83d51ba0%2C38659%2C1732104813354.meta:.meta(num 1732104819903) 2024-11-20T12:13:43,108 DEBUG [RS:0;843d83d51ba0:44663 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/oldWALs 2024-11-20T12:13:43,108 INFO [RS:0;843d83d51ba0:44663 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 843d83d51ba0%2C44663%2C1732104813203:(num 1732104814863) 2024-11-20T12:13:43,108 DEBUG [RS:0;843d83d51ba0:44663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:13:43,108 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T12:13:43,108 INFO [RS:0;843d83d51ba0:44663 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T12:13:43,108 INFO [RS:0;843d83d51ba0:44663 {}] hbase.ChoreService(370): Chore service for: regionserver/843d83d51ba0:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T12:13:43,109 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-20T12:13:43,109 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-20T12:13:43,109 INFO [regionserver/843d83d51ba0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T12:13:43,109 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-20T12:13:43,109 INFO [RS:0;843d83d51ba0:44663 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T12:13:43,109 INFO [RS:0;843d83d51ba0:44663 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44663 2024-11-20T12:13:43,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741833_1009 (size=7074) 2024-11-20T12:13:43,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741833_1009 (size=7074) 2024-11-20T12:13:43,118 DEBUG [RS:2;843d83d51ba0:38659 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/oldWALs 2024-11-20T12:13:43,118 INFO [RS:2;843d83d51ba0:38659 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 843d83d51ba0%2C38659%2C1732104813354:(num 1732104814863) 2024-11-20T12:13:43,118 DEBUG [RS:2;843d83d51ba0:38659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-20T12:13:43,118 INFO [RS:2;843d83d51ba0:38659 {}] regionserver.LeaseManager(133): Closed leases 2024-11-20T12:13:43,118 INFO [RS:2;843d83d51ba0:38659 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T12:13:43,118 INFO [RS:2;843d83d51ba0:38659 {}] hbase.ChoreService(370): Chore service for: regionserver/843d83d51ba0:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-20T12:13:43,118 INFO [RS:2;843d83d51ba0:38659 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T12:13:43,118 INFO [regionserver/843d83d51ba0:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-20T12:13:43,118 INFO [RS:2;843d83d51ba0:38659 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:38659 2024-11-20T12:13:43,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T12:13:43,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/843d83d51ba0,44663,1732104813203 2024-11-20T12:13:43,119 INFO [RS:0;843d83d51ba0:44663 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T12:13:43,129 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [843d83d51ba0,44663,1732104813203] 2024-11-20T12:13:43,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38659-0x1015829ee370003, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/843d83d51ba0,38659,1732104813354 2024-11-20T12:13:43,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-20T12:13:43,140 INFO [RS:2;843d83d51ba0:38659 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-20T12:13:43,150 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/843d83d51ba0,44663,1732104813203 already deleted, retry=false 2024-11-20T12:13:43,151 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 843d83d51ba0,44663,1732104813203 expired; onlineServers=1 2024-11-20T12:13:43,161 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [843d83d51ba0,38659,1732104813354] 2024-11-20T12:13:43,171 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/843d83d51ba0,38659,1732104813354 already deleted, retry=false 2024-11-20T12:13:43,171 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 843d83d51ba0,38659,1732104813354 expired; onlineServers=0 2024-11-20T12:13:43,172 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '843d83d51ba0,39157,1732104812301' ***** 2024-11-20T12:13:43,172 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-20T12:13:43,172 INFO [M:0;843d83d51ba0:39157 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-20T12:13:43,172 INFO [M:0;843d83d51ba0:39157 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-20T12:13:43,172 DEBUG [M:0;843d83d51ba0:39157 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-20T12:13:43,172 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-20T12:13:43,172 DEBUG [M:0;843d83d51ba0:39157 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-20T12:13:43,172 DEBUG [master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.small.0-1732104814499 {}] cleaner.HFileCleaner(306): Exit Thread[master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.small.0-1732104814499,5,FailOnTimeoutGroup] 2024-11-20T12:13:43,172 DEBUG [master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.large.0-1732104814498 {}] cleaner.HFileCleaner(306): Exit Thread[master/843d83d51ba0:0:becomeActiveMaster-HFileCleaner.large.0-1732104814498,5,FailOnTimeoutGroup] 2024-11-20T12:13:43,173 INFO [M:0;843d83d51ba0:39157 {}] hbase.ChoreService(370): Chore service for: master/843d83d51ba0:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-20T12:13:43,173 INFO [M:0;843d83d51ba0:39157 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-20T12:13:43,173 DEBUG [M:0;843d83d51ba0:39157 {}] master.HMaster(1795): Stopping service threads 2024-11-20T12:13:43,173 INFO [M:0;843d83d51ba0:39157 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-20T12:13:43,173 INFO [M:0;843d83d51ba0:39157 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-20T12:13:43,173 INFO [M:0;843d83d51ba0:39157 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-20T12:13:43,173 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-20T12:13:43,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-20T12:13:43,182 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-20T12:13:43,182 DEBUG [M:0;843d83d51ba0:39157 {}] zookeeper.ZKUtil(347): master:39157-0x1015829ee370000, quorum=127.0.0.1:56800, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-20T12:13:43,182 WARN [M:0;843d83d51ba0:39157 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-20T12:13:43,183 INFO [M:0;843d83d51ba0:39157 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/.lastflushedseqids 2024-11-20T12:13:43,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741870_1046 (size=1832) 2024-11-20T12:13:43,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741870_1046 (size=1832) 2024-11-20T12:13:43,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741870_1046 (size=1832) 2024-11-20T12:13:43,197 INFO [M:0;843d83d51ba0:39157 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-20T12:13:43,197 INFO [M:0;843d83d51ba0:39157 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-20T12:13:43,197 DEBUG [M:0;843d83d51ba0:39157 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-20T12:13:43,197 INFO [M:0;843d83d51ba0:39157 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:13:43,197 DEBUG [M:0;843d83d51ba0:39157 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:13:43,197 DEBUG [M:0;843d83d51ba0:39157 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-20T12:13:43,197 DEBUG [M:0;843d83d51ba0:39157 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-20T12:13:43,198 INFO [M:0;843d83d51ba0:39157 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=260.14 KB heapSize=316.21 KB 2024-11-20T12:13:43,216 DEBUG [M:0;843d83d51ba0:39157 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c6957a1d0c184f9e8d0f8597ba263998 is 82, key is hbase:meta,,1/info:regioninfo/1732104820222/Put/seqid=0 2024-11-20T12:13:43,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34249 is added to blk_1073741871_1047 (size=6071) 2024-11-20T12:13:43,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38431 is added to blk_1073741871_1047 (size=6071) 2024-11-20T12:13:43,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41703 is added to blk_1073741871_1047 (size=6071) 2024-11-20T12:13:43,225 INFO [M:0;843d83d51ba0:39157 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=601 (bloomFilter=true), to=hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c6957a1d0c184f9e8d0f8597ba263998 2024-11-20T12:13:43,230 INFO [RS:0;843d83d51ba0:44663 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-20T12:13:43,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T12:13:43,230 INFO [RS:0;843d83d51ba0:44663 {}] regionserver.HRegionServer(1031): Exiting; stopping=843d83d51ba0,44663,1732104813203; zookeeper connection closed. 2024-11-20T12:13:43,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44663-0x1015829ee370001, quorum=127.0.0.1:56800, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-20T12:13:43,230 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@19564e30 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@19564e30 2024-11-20T12:13:43,252 DEBUG [M:0;843d83d51ba0:39157 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:42641/user/jenkins/test-data/dcb5eee2-e0a7-5991-a469-6526c6b88dad/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/17edd625f7a24f92bbf627168436c6d2 is 2987, key is \x00\x00\x00\x00\x00\x00\x00