2024-12-14 20:33:57,160 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5e7cd6cc 2024-12-14 20:33:57,170 main DEBUG Took 0.008384 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-14 20:33:57,170 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-14 20:33:57,170 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-14 20:33:57,171 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-14 20:33:57,172 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,178 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-14 20:33:57,188 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,190 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,190 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,191 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,191 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,192 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,192 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,192 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,193 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,193 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,194 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,194 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,194 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,195 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,195 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,195 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,196 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,196 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,196 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,196 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,197 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,197 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-14 20:33:57,197 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,198 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-14 20:33:57,199 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-14 20:33:57,200 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-14 20:33:57,202 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-14 20:33:57,202 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-14 20:33:57,203 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-14 20:33:57,204 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-14 20:33:57,211 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-14 20:33:57,214 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-14 20:33:57,216 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-14 20:33:57,216 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-14 20:33:57,216 main DEBUG createAppenders(={Console}) 2024-12-14 20:33:57,217 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5e7cd6cc initialized 2024-12-14 20:33:57,217 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5e7cd6cc 2024-12-14 20:33:57,218 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@5e7cd6cc OK. 2024-12-14 20:33:57,218 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-14 20:33:57,218 main DEBUG OutputStream closed 2024-12-14 20:33:57,219 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-14 20:33:57,219 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-14 20:33:57,219 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@434a63ab OK 2024-12-14 20:33:57,280 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-14 20:33:57,282 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-14 20:33:57,284 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-14 20:33:57,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-14 20:33:57,285 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-14 20:33:57,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-14 20:33:57,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-14 20:33:57,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-14 20:33:57,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-14 20:33:57,286 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-14 20:33:57,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-14 20:33:57,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-14 20:33:57,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-14 20:33:57,287 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-14 20:33:57,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-14 20:33:57,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-14 20:33:57,288 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-14 20:33:57,289 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-14 20:33:57,291 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-14 20:33:57,291 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@8c3619e) with optional ClassLoader: null 2024-12-14 20:33:57,292 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-14 20:33:57,292 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@8c3619e] started OK. 2024-12-14T20:33:57,305 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen timeout: 13 mins 2024-12-14 20:33:57,308 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-14 20:33:57,308 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-14T20:33:57,349 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestAMServerFailedOpen#testCallQueueTooBigExceptionOnAssignment Thread=11, OpenFileDescriptor=282, MaxFileDescriptor=1048576, SystemLoadAverage=110, ProcessCount=11, AvailableMemoryMB=7080 2024-12-14T20:33:57,575 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a 2024-12-14T20:33:58,405 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-14T20:33:58,440 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-14T20:33:58,503 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-14T20:33:58,696 INFO [Time-limited test {}] util.FSUtils(490): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a with version=8 2024-12-14T20:33:58,721 DEBUG [Time-limited test {}] util.FSUtils(639): Created cluster ID file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/hbase.id with ID: da12e6ed-7f3e-49a3-bc1e-e7c7dc7e76c4 2024-12-14T20:33:58,758 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-14T20:33:58,782 INFO [Time-limited test {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-14T20:33:58,785 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-14T20:33:58,857 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.MockMasterServices.(MockMasterServices.java:120) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase.setUp(TestAssignmentManagerBase.java:157) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:33:58,861 INFO [Time-limited test {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-14T20:33:58,865 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-14T20:33:58,877 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-14T20:33:58,877 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-14T20:33:58,929 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-14T20:33:58,934 INFO [Time-limited test {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/MasterData/data/master/store 2024-12-14T20:33:58,947 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-14T20:33:58,948 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-14T20:33:58,997 DEBUG [Time-limited test {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-14T20:33:58,997 INFO [Time-limited test {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:33:58,997 DEBUG [Time-limited test {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:33:58,997 DEBUG [Time-limited test {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-14T20:33:58,998 DEBUG [Time-limited test {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:33:58,998 INFO [Time-limited test {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:33:58,998 DEBUG [Time-limited test {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-14T20:33:58,998 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/MasterData/data/master/store/.initializing 2024-12-14T20:33:58,999 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/MasterData/WALs/mockmaster.example.org,1234,-1 2024-12-14T20:33:59,009 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-14T20:33:59,019 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/MasterData/oldWALs, maxLogs=10 2024-12-14T20:33:59,085 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1734208439049 2024-12-14T20:33:59,086 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [] 2024-12-14T20:33:59,086 DEBUG [Time-limited test {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-14T20:33:59,087 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-14T20:33:59,091 DEBUG [Time-limited test {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:33:59,092 DEBUG [Time-limited test {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:33:59,124 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:33:59,150 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-14T20:33:59,154 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:33:59,157 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-14T20:33:59,158 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:33:59,161 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-14T20:33:59,161 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:33:59,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-14T20:33:59,162 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:33:59,165 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-14T20:33:59,165 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:33:59,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-14T20:33:59,166 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:33:59,170 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-14T20:33:59,170 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:33:59,171 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-14T20:33:59,176 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:33:59,177 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:33:59,183 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-14T20:33:59,186 DEBUG [Time-limited test {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:33:59,194 DEBUG [Time-limited test {}] wal.WALSplitUtil(409): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-14T20:33:59,194 INFO [Time-limited test {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70164681, jitterRate=0.04553522169589996}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-14T20:33:59,199 DEBUG [Time-limited test {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-14T20:33:59,200 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-14T20:33:59,486 INFO [Time-limited test {}] procedure2.ProcedureExecutor(633): Starting 64 core workers (bigger of cpus/4 or 16) with max (burst) worker count=640 2024-12-14T20:33:59,489 INFO [Time-limited test {}] procedure2.ProcedureExecutor(653): Recovered NoopProcedureStore lease in 0 msec 2024-12-14T20:33:59,490 INFO [Time-limited test {}] procedure2.ProcedureExecutor(667): Loaded NoopProcedureStore in 0 msec 2024-12-14T20:33:59,515 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734208469514 2024-12-14T20:33:59,515 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-14T20:33:59,546 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,100,1 2024-12-14T20:33:59,549 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,101,1 2024-12-14T20:33:59,549 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,102,1 2024-12-14T20:33:59,549 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,103,1 2024-12-14T20:33:59,550 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,104,1 2024-12-14T20:33:59,550 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,105,1 2024-12-14T20:33:59,550 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,106,1 2024-12-14T20:33:59,550 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,107,1 2024-12-14T20:33:59,551 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,108,1 2024-12-14T20:33:59,551 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,109,1 2024-12-14T20:33:59,558 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-14T20:33:59,562 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-14T20:33:59,563 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-14T20:33:59,611 DEBUG [Time-limited test {}] procedure.ProcedureSyncWait(219): waitFor pid=1 2024-12-14T20:33:59,717 DEBUG [mockmaster:1234 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=10, allServersCount=10 2024-12-14T20:33:59,720 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:33:59,722 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:33:59,722 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:33:59,722 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:33:59,722 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:33:59,723 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:33:59,723 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:33:59,723 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:33:59,723 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:33:59,723 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:33:59,723 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:33:59,723 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:33:59,724 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:33:59,724 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:33:59,724 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:33:59,724 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:33:59,724 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:33:59,724 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:33:59,724 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:33:59,724 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:33:59,725 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:33:59,725 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:33:59,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE; OpenRegionProcedure 1588230740, server=localhost,109,1}] 2024-12-14T20:33:59,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-14T20:33:59,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; OpenRegionProcedure 1588230740, server=localhost,109,1 in 179 msec 2024-12-14T20:33:59,908 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 349 msec 2024-12-14T20:33:59,914 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1098): Stored pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN 2024-12-14T20:33:59,915 INFO [PEWorker-6 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN 2024-12-14T20:33:59,915 INFO [PEWorker-6 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-14T20:33:59,965 DEBUG [Time-limited test {}] procedure.ProcedureSyncWait(219): waitFor pid=3 2024-12-14T20:34:00,065 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:00,066 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:00,066 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:00,066 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:00,066 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:00,066 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:00,066 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:00,067 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:00,067 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:00,067 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:00,067 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:00,067 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:00,067 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:00,067 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:00,067 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:00,067 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:00,068 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:00,068 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:00,068 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:00,068 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:00,068 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:00,068 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:00,069 INFO [PEWorker-7 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=4, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,106,1}] 2024-12-14T20:34:00,220 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,106,1 failed, try=0 org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:00,221 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(348): request to localhost,106,1 failed due to org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault, try=0, this usually because server is overloaded, give up 2024-12-14T20:34:00,221 WARN [MockRSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=4, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,106,1 for region state=OPENING, location=localhost,106,1, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 to server localhost,106,1 failed org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:00,222 INFO [PEWorker-9 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=4, resume processing ppid=3 2024-12-14T20:34:00,222 INFO [PEWorker-9 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, ppid=3, state=SUCCESS; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,106,1 in 154 msec 2024-12-14T20:34:00,223 INFO [PEWorker-10 {}] assignment.TransitRegionStateProcedure(308): Retry=1 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=localhost,106,1 2024-12-14T20:34:00,223 INFO [PEWorker-10 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:00,374 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:00,375 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:00,375 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:00,375 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:00,375 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:00,376 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:00,376 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:00,376 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:00,376 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:00,377 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:00,377 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:00,377 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:00,377 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:00,378 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:00,378 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:00,378 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:00,378 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:00,379 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:00,379 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:00,379 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:00,380 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:00,380 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:00,381 INFO [PEWorker-11 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,107,1}] 2024-12-14T20:34:00,533 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=0 org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:00,535 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(348): request to localhost,107,1 failed due to org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault, try=0, this usually because server is overloaded, give up 2024-12-14T20:34:00,535 WARN [MockRSProcedureDispatcher-pool-2 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=5, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,107,1 for region state=OPENING, location=localhost,107,1, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 to server localhost,107,1 failed org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:00,537 INFO [PEWorker-13 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=3 2024-12-14T20:34:00,537 INFO [PEWorker-14 {}] assignment.TransitRegionStateProcedure(308): Retry=2 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=localhost,107,1 2024-12-14T20:34:00,537 INFO [PEWorker-13 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=3, state=SUCCESS; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,107,1 in 157 msec 2024-12-14T20:34:00,538 INFO [PEWorker-14 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:00,688 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:00,689 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:00,689 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:00,689 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:00,689 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:00,689 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:00,689 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:00,689 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:00,689 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:00,689 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:00,689 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:00,690 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:00,690 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:00,690 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:00,690 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:00,690 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:00,690 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:00,691 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:00,691 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:00,691 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:00,691 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:00,691 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:00,692 INFO [PEWorker-15 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,103,1}] 2024-12-14T20:34:00,843 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=0 org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:00,843 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(348): request to localhost,103,1 failed due to org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault, try=0, this usually because server is overloaded, give up 2024-12-14T20:34:00,844 WARN [MockRSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=6, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,103,1 for region state=OPENING, location=localhost,103,1, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 to server localhost,103,1 failed org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:00,845 INFO [PEWorker-17 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=3 2024-12-14T20:34:00,845 INFO [PEWorker-18 {}] assignment.TransitRegionStateProcedure(308): Retry=3 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=localhost,103,1 2024-12-14T20:34:00,845 INFO [PEWorker-17 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=3, state=SUCCESS; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,103,1 in 153 msec 2024-12-14T20:34:00,845 INFO [PEWorker-18 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:00,996 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:00,997 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:00,997 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:00,998 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:00,998 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:00,998 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:00,999 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:00,999 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:01,000 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:01,000 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:01,001 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:01,001 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:01,002 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:01,002 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:01,003 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:01,003 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:01,004 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:01,004 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:01,005 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:01,005 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:01,005 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:01,006 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:01,007 INFO [PEWorker-19 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=7, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,105,1}] 2024-12-14T20:34:01,158 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=0 org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:01,160 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(348): request to localhost,105,1 failed due to org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault, try=0, this usually because server is overloaded, give up 2024-12-14T20:34:01,161 WARN [MockRSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=7, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,105,1 for region state=OPENING, location=localhost,105,1, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 to server localhost,105,1 failed org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:01,163 INFO [PEWorker-21 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=7, resume processing ppid=3 2024-12-14T20:34:01,163 INFO [PEWorker-22 {}] assignment.TransitRegionStateProcedure(308): Retry=4 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=localhost,105,1 2024-12-14T20:34:01,163 INFO [PEWorker-21 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, ppid=3, state=SUCCESS; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,105,1 in 156 msec 2024-12-14T20:34:01,166 WARN [PEWorker-22 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:01,167 INFO [PEWorker-22 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; timeout=10, timestamp=1734208441177 2024-12-14T20:34:01,177 INFO [PEWorker-23 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:01,328 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:01,329 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:01,329 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:01,329 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:01,330 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:01,330 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:01,330 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:01,331 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:01,331 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:01,331 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:01,331 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:01,332 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:01,332 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:01,332 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:01,333 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:01,333 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:01,333 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:01,333 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:01,334 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:01,334 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:01,334 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:01,334 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:01,335 INFO [PEWorker-24 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=8, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,107,1}] 2024-12-14T20:34:01,487 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=0 org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:01,488 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(348): request to localhost,107,1 failed due to org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault, try=0, this usually because server is overloaded, give up 2024-12-14T20:34:01,489 WARN [MockRSProcedureDispatcher-pool-2 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=8, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,107,1 for region state=OPENING, location=localhost,107,1, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 to server localhost,107,1 failed org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:01,491 INFO [PEWorker-26 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=8, resume processing ppid=3 2024-12-14T20:34:01,491 INFO [PEWorker-27 {}] assignment.TransitRegionStateProcedure(308): Retry=5 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=localhost,107,1 2024-12-14T20:34:01,491 INFO [PEWorker-26 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, ppid=3, state=SUCCESS; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,107,1 in 155 msec 2024-12-14T20:34:01,492 WARN [PEWorker-27 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:01,492 INFO [PEWorker-27 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; timeout=20, timestamp=1734208441512 2024-12-14T20:34:01,512 INFO [PEWorker-28 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:01,663 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:01,664 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:01,664 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:01,665 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:01,665 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:01,665 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:01,666 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:01,666 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:01,666 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:01,667 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:01,667 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:01,667 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:01,668 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:01,668 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:01,668 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:01,669 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:01,669 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:01,669 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:01,670 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:01,670 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:01,670 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:01,671 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:01,672 INFO [PEWorker-29 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=9, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,101,1}] 2024-12-14T20:34:01,824 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=0 org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:01,826 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(348): request to localhost,101,1 failed due to org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault, try=0, this usually because server is overloaded, give up 2024-12-14T20:34:01,826 WARN [MockRSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=9, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,101,1 for region state=OPENING, location=localhost,101,1, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 to server localhost,101,1 failed org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:01,828 INFO [PEWorker-31 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=9, resume processing ppid=3 2024-12-14T20:34:01,829 INFO [PEWorker-32 {}] assignment.TransitRegionStateProcedure(308): Retry=6 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=localhost,101,1 2024-12-14T20:34:01,829 INFO [PEWorker-31 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, ppid=3, state=SUCCESS; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,101,1 in 156 msec 2024-12-14T20:34:01,829 WARN [PEWorker-32 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:01,831 INFO [PEWorker-32 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; timeout=40, timestamp=1734208441870 2024-12-14T20:34:01,871 INFO [PEWorker-33 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:02,021 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:02,022 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:02,022 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:02,023 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:02,023 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:02,023 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:02,024 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:02,024 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:02,024 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:02,025 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:02,025 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:02,025 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:02,025 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:02,026 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:02,026 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:02,026 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:02,027 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:02,027 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:02,027 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:02,027 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:02,027 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:02,027 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:02,028 INFO [PEWorker-34 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,104,1}] 2024-12-14T20:34:02,179 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=0 org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:02,180 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(348): request to localhost,104,1 failed due to org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault, try=0, this usually because server is overloaded, give up 2024-12-14T20:34:02,180 WARN [MockRSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=10, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,104,1 for region state=OPENING, location=localhost,104,1, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 to server localhost,104,1 failed org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:02,181 INFO [PEWorker-36 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=3 2024-12-14T20:34:02,182 INFO [PEWorker-36 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=3, state=SUCCESS; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,104,1 in 153 msec 2024-12-14T20:34:02,182 INFO [PEWorker-37 {}] assignment.TransitRegionStateProcedure(308): Retry=7 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=localhost,104,1 2024-12-14T20:34:02,182 WARN [PEWorker-37 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:02,183 INFO [PEWorker-37 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; timeout=80, timestamp=1734208442263 2024-12-14T20:34:02,264 INFO [PEWorker-38 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:02,415 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:02,415 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:02,416 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:02,416 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:02,416 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:02,416 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:02,417 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:02,417 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:02,417 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:02,417 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:02,418 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:02,418 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:02,418 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:02,418 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:02,419 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:02,419 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:02,419 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:02,420 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:02,420 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:02,420 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:02,420 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:02,421 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:02,422 INFO [PEWorker-39 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,105,1}] 2024-12-14T20:34:02,573 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=0 org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:02,574 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(348): request to localhost,105,1 failed due to org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault, try=0, this usually because server is overloaded, give up 2024-12-14T20:34:02,575 WARN [MockRSProcedureDispatcher-pool-2 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=11, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,105,1 for region state=OPENING, location=localhost,105,1, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 to server localhost,105,1 failed org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:02,577 INFO [PEWorker-41 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=3 2024-12-14T20:34:02,577 INFO [PEWorker-42 {}] assignment.TransitRegionStateProcedure(308): Retry=8 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=localhost,105,1 2024-12-14T20:34:02,577 INFO [PEWorker-41 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=3, state=SUCCESS; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,105,1 in 155 msec 2024-12-14T20:34:02,578 WARN [PEWorker-42 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:02,578 INFO [PEWorker-42 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; timeout=100, timestamp=1734208442678 2024-12-14T20:34:02,679 INFO [PEWorker-43 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:02,831 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:02,831 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:02,831 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:02,832 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:02,832 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:02,832 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:02,832 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:02,832 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:02,833 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:02,833 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:02,833 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:02,833 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:02,834 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:02,834 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:02,834 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:02,834 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:02,835 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:02,835 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:02,835 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:02,835 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:02,835 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:02,836 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:02,837 INFO [PEWorker-44 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=12, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,109,1}] 2024-12-14T20:34:02,988 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,109,1 failed, try=0 org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:02,990 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(348): request to localhost,109,1 failed due to org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault, try=0, this usually because server is overloaded, give up 2024-12-14T20:34:02,990 WARN [MockRSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=12, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,109,1 for region state=OPENING, location=localhost,109,1, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 to server localhost,109,1 failed org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:02,992 INFO [PEWorker-46 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=12, resume processing ppid=3 2024-12-14T20:34:02,992 INFO [PEWorker-47 {}] assignment.TransitRegionStateProcedure(308): Retry=9 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=localhost,109,1 2024-12-14T20:34:02,992 INFO [PEWorker-46 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, ppid=3, state=SUCCESS; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,109,1 in 155 msec 2024-12-14T20:34:02,993 WARN [PEWorker-47 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:02,994 INFO [PEWorker-47 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; timeout=100, timestamp=1734208443094 2024-12-14T20:34:03,095 INFO [PEWorker-48 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:03,246 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:03,247 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:03,247 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:03,247 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:03,248 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:03,248 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:03,248 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:03,248 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:03,249 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:03,249 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:03,249 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:03,249 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:03,249 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:03,250 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:03,250 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:03,250 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:03,250 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:03,250 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:03,251 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:03,251 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:03,251 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:03,251 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:03,252 INFO [PEWorker-49 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,102,1}] 2024-12-14T20:34:03,403 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,102,1 failed, try=0 org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:03,403 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(348): request to localhost,102,1 failed due to org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault, try=0, this usually because server is overloaded, give up 2024-12-14T20:34:03,404 WARN [MockRSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=13, ppid=3, state=RUNNABLE; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,102,1 for region state=OPENING, location=localhost,102,1, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 to server localhost,102,1 failed org.apache.hadoop.hbase.CallQueueTooBigException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:03,405 INFO [PEWorker-51 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=3 2024-12-14T20:34:03,405 INFO [PEWorker-52 {}] assignment.TransitRegionStateProcedure(308): Retry=10 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN; state=OPENING, location=localhost,102,1 2024-12-14T20:34:03,405 INFO [PEWorker-51 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=3, state=SUCCESS; OpenRegionProcedure 8a268d1a0948fedcd7b7d6332c8390f9, server=localhost,102,1 in 153 msec 2024-12-14T20:34:03,406 ERROR [PEWorker-52 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=3, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.client.RetriesExhaustedException via TransitRegionStateProcedure:org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:312) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:313) ~[classes/:?] ... 12 more 2024-12-14T20:34:03,407 INFO [PEWorker-52 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=3, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.client.RetriesExhaustedException via TransitRegionStateProcedure:org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded; TransitRegionStateProcedure table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9, ASSIGN exec-time=3.4930 sec 2024-12-14T20:34:03,407 WARN [PEWorker-52 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-14T20:34:03,424 INFO [Time-limited test {}] assignment.TestAssignmentManagerBase(221): ExecutionException java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait$ProcedureFuture.get(ProcedureSyncWait.java:117) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait$ProcedureFuture.get(ProcedureSyncWait.java:65) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase.waitOnFuture(TestAssignmentManagerBase.java:219) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testFailedOpen(TestAMServerFailedOpen.java:99) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testCallQueueTooBigExceptionOnAssignment(TestAMServerFailedOpen.java:114) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:313) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:03,425 INFO [Time-limited test {}] assignment.TestAMServerFailedOpen(103): REGION STATE state=FAILED_OPEN, location=null, table=testCallQueueTooBigExceptionOnAssignment, region=8a268d1a0948fedcd7b7d6332c8390f9 2024-12-14T20:34:03,425 INFO [Time-limited test {}] assignment.TestAMServerFailedOpen(104): expected exception from assign operation: Max attempts 10 exceeded org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:313) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:03,427 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-14T20:34:03,428 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-14T20:34:03,428 INFO [Time-limited test {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-14T20:34:03,438 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestAMServerFailedOpen#testCallQueueTooBigExceptionOnAssignment Thread=23 (was 11) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: process reaper java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:401) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1734208439049 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: master:store-Flusher java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759) app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor.flushLoop(MasterRegionFlusherAndCompactor.java:200) app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor$$Lambda$299/0x00007f6b2c4d06a0.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master:store-WAL-Roller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:179) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=288 (was 282) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=101 (was 110), ProcessCount=11 (was 11), AvailableMemoryMB=6898 (was 7080) 2024-12-14T20:34:03,443 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestAMServerFailedOpen#testServerNotYetRunning Thread=23, OpenFileDescriptor=288, MaxFileDescriptor=1048576, SystemLoadAverage=101, ProcessCount=11, AvailableMemoryMB=6898 2024-12-14T20:34:03,454 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/hadoop-log-dir so I do NOT create it in target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7 2024-12-14T20:34:03,455 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/hadoop-log-dir Erasing configuration value by system value. 2024-12-14T20:34:03,455 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/hadoop-tmp-dir so I do NOT create it in target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7 2024-12-14T20:34:03,455 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/hadoop-tmp-dir Erasing configuration value by system value. 2024-12-14T20:34:03,455 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7 2024-12-14T20:34:03,466 INFO [Time-limited test {}] util.FSUtils(490): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7 with version=8 2024-12-14T20:34:03,471 DEBUG [Time-limited test {}] util.FSUtils(639): Created cluster ID file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/hbase.id with ID: 071fbf47-4769-4f82-a212-2c013c241c39 2024-12-14T20:34:03,496 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-14T20:34:03,496 INFO [Time-limited test {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-14T20:34:03,499 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-14T20:34:03,499 INFO [Time-limited test {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-14T20:34:03,519 INFO [Time-limited test {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/MasterData/data/master/store 2024-12-14T20:34:03,527 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-14T20:34:03,527 DEBUG [Time-limited test {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-14T20:34:03,527 INFO [Time-limited test {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:34:03,527 DEBUG [Time-limited test {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:34:03,527 DEBUG [Time-limited test {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-14T20:34:03,527 DEBUG [Time-limited test {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:34:03,527 INFO [Time-limited test {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:34:03,528 DEBUG [Time-limited test {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-14T20:34:03,528 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/MasterData/data/master/store/.initializing 2024-12-14T20:34:03,528 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/MasterData/WALs/mockmaster.example.org,1234,-1 2024-12-14T20:34:03,533 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-14T20:34:03,537 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/MasterData/oldWALs, maxLogs=10 2024-12-14T20:34:03,556 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1734208443537 2024-12-14T20:34:03,556 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [] 2024-12-14T20:34:03,557 DEBUG [Time-limited test {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-14T20:34:03,557 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-14T20:34:03,557 DEBUG [Time-limited test {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:03,557 DEBUG [Time-limited test {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:03,558 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:03,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-14T20:34:03,561 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:34:03,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-14T20:34:03,562 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:03,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-14T20:34:03,565 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:34:03,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-14T20:34:03,566 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:03,569 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-14T20:34:03,569 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:34:03,570 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-14T20:34:03,570 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:03,573 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-14T20:34:03,574 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:34:03,574 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-14T20:34:03,575 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:03,576 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:03,577 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-14T20:34:03,578 DEBUG [Time-limited test {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:03,585 DEBUG [Time-limited test {}] wal.WALSplitUtil(409): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-14T20:34:03,586 INFO [Time-limited test {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60427784, jitterRate=-0.0995558500289917}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-14T20:34:03,586 DEBUG [Time-limited test {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-14T20:34:03,586 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-14T20:34:03,594 INFO [Time-limited test {}] procedure2.ProcedureExecutor(633): Starting 64 core workers (bigger of cpus/4 or 16) with max (burst) worker count=640 2024-12-14T20:34:03,595 INFO [Time-limited test {}] procedure2.ProcedureExecutor(653): Recovered NoopProcedureStore lease in 0 msec 2024-12-14T20:34:03,596 INFO [Time-limited test {}] procedure2.ProcedureExecutor(667): Loaded NoopProcedureStore in 0 msec 2024-12-14T20:34:03,600 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734208473600 2024-12-14T20:34:03,601 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-14T20:34:03,601 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,100,1 2024-12-14T20:34:03,601 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,101,1 2024-12-14T20:34:03,601 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,102,1 2024-12-14T20:34:03,602 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,103,1 2024-12-14T20:34:03,602 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,104,1 2024-12-14T20:34:03,602 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,105,1 2024-12-14T20:34:03,602 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,106,1 2024-12-14T20:34:03,602 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,107,1 2024-12-14T20:34:03,602 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,108,1 2024-12-14T20:34:03,602 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,109,1 2024-12-14T20:34:03,602 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-14T20:34:03,603 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-14T20:34:03,603 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-14T20:34:03,653 DEBUG [Time-limited test {}] procedure.ProcedureSyncWait(219): waitFor pid=1 2024-12-14T20:34:03,754 DEBUG [mockmaster:1234 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=10, allServersCount=10 2024-12-14T20:34:03,754 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:03,755 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:03,755 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:03,755 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:03,756 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:03,756 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:03,756 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:03,756 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:03,757 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:03,757 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:03,757 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:03,757 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:03,757 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:03,757 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:03,757 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:03,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:03,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:03,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:03,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:03,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:03,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:03,759 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:03,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE; OpenRegionProcedure 1588230740, server=localhost,106,1}] 2024-12-14T20:34:03,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-14T20:34:03,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; OpenRegionProcedure 1588230740, server=localhost,106,1 in 153 msec 2024-12-14T20:34:03,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 310 msec 2024-12-14T20:34:03,955 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1098): Stored pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN 2024-12-14T20:34:03,956 INFO [PEWorker-6 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN 2024-12-14T20:34:03,957 INFO [PEWorker-6 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-14T20:34:04,006 DEBUG [Time-limited test {}] procedure.ProcedureSyncWait(219): waitFor pid=3 2024-12-14T20:34:04,107 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:04,108 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:04,108 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:04,109 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:04,109 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:04,109 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:04,109 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:04,109 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:04,109 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:04,110 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:04,110 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:04,110 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:04,110 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:04,110 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:04,110 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:04,111 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:04,111 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:04,111 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:04,111 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:04,111 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:04,112 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:04,112 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:04,113 INFO [PEWorker-7 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=4, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,103,1}] 2024-12-14T20:34:04,264 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=0 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:04,265 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=0, can wait up to 1000ms 2024-12-14T20:34:04,367 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=1 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:04,368 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=1, can wait up to 897ms 2024-12-14T20:34:04,469 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=2 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:04,470 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=2, can wait up to 795ms 2024-12-14T20:34:04,570 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=3 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:04,571 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=3, can wait up to 694ms 2024-12-14T20:34:04,672 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=4 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:04,672 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=4, can wait up to 593ms 2024-12-14T20:34:04,773 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=5 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:04,774 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=5, can wait up to 491ms 2024-12-14T20:34:04,875 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=6 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:04,876 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=6, can wait up to 389ms 2024-12-14T20:34:04,977 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=7 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:04,977 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=7, can wait up to 288ms 2024-12-14T20:34:05,078 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=8 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:05,079 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=8, can wait up to 186ms 2024-12-14T20:34:05,180 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=9 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:05,181 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=9, can wait up to 84ms 2024-12-14T20:34:05,282 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=10 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:05,283 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): localhost,103,1 is throwing ServerNotRunningYetException for 1734208445265ms; trying another server 2024-12-14T20:34:05,283 WARN [MockRSProcedureDispatcher-pool-2 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=4, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,103,1 for region state=OPENING, location=localhost,103,1, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 to server localhost,103,1 failed org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:05,284 INFO [PEWorker-9 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=4, resume processing ppid=3 2024-12-14T20:34:05,284 INFO [PEWorker-10 {}] assignment.TransitRegionStateProcedure(308): Retry=1 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=localhost,103,1 2024-12-14T20:34:05,284 INFO [PEWorker-9 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, ppid=3, state=SUCCESS; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,103,1 in 1.1710 sec 2024-12-14T20:34:05,284 INFO [PEWorker-10 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:05,435 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:05,436 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:05,436 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:05,436 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:05,436 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:05,436 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:05,436 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:05,437 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:05,437 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:05,437 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:05,437 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:05,437 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:05,437 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:05,437 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:05,438 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:05,438 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:05,438 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:05,438 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:05,438 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:05,438 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:05,438 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:05,439 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:05,440 INFO [PEWorker-11 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,100,1}] 2024-12-14T20:34:05,591 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=0 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:05,592 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=0, can wait up to 1000ms 2024-12-14T20:34:05,693 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=1 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:05,693 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=1, can wait up to 899ms 2024-12-14T20:34:05,794 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=2 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:05,795 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=2, can wait up to 797ms 2024-12-14T20:34:05,895 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=3 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:05,896 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=3, can wait up to 696ms 2024-12-14T20:34:05,997 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=4 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:05,998 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=4, can wait up to 594ms 2024-12-14T20:34:06,099 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=5 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:06,100 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=5, can wait up to 492ms 2024-12-14T20:34:06,201 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=6 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:06,201 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=6, can wait up to 391ms 2024-12-14T20:34:06,301 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=7 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:06,302 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=7, can wait up to 290ms 2024-12-14T20:34:06,403 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=8 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:06,404 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=8, can wait up to 189ms 2024-12-14T20:34:06,505 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=9 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:06,505 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=9, can wait up to 87ms 2024-12-14T20:34:06,606 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=10 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:06,607 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): localhost,100,1 is throwing ServerNotRunningYetException for 1734208446592ms; trying another server 2024-12-14T20:34:06,607 WARN [MockRSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=5, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,100,1 for region state=OPENING, location=localhost,100,1, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 to server localhost,100,1 failed org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:06,609 INFO [PEWorker-13 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=3 2024-12-14T20:34:06,609 INFO [PEWorker-14 {}] assignment.TransitRegionStateProcedure(308): Retry=2 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=localhost,100,1 2024-12-14T20:34:06,610 INFO [PEWorker-13 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=3, state=SUCCESS; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,100,1 in 1.1700 sec 2024-12-14T20:34:06,610 INFO [PEWorker-14 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:06,761 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:06,761 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:06,762 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:06,762 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:06,762 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:06,762 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:06,763 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:06,763 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:06,763 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:06,763 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:06,763 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:06,763 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:06,764 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:06,764 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:06,764 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:06,764 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:06,764 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:06,764 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:06,764 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:06,765 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:06,765 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:06,765 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:06,766 INFO [PEWorker-15 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,100,1}] 2024-12-14T20:34:06,917 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=0 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:06,918 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=0, can wait up to 1000ms 2024-12-14T20:34:07,019 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=1 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:07,019 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=1, can wait up to 899ms 2024-12-14T20:34:07,120 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=2 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:07,121 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=2, can wait up to 797ms 2024-12-14T20:34:07,222 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=3 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:07,223 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=3, can wait up to 695ms 2024-12-14T20:34:07,324 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=4 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:07,325 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=4, can wait up to 594ms 2024-12-14T20:34:07,426 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=5 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:07,426 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=5, can wait up to 492ms 2024-12-14T20:34:07,526 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=6 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:07,527 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=6, can wait up to 391ms 2024-12-14T20:34:07,628 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=7 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:07,629 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=7, can wait up to 290ms 2024-12-14T20:34:07,730 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=8 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:07,730 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=8, can wait up to 188ms 2024-12-14T20:34:07,831 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=9 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:07,832 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,100,1, try=9, can wait up to 86ms 2024-12-14T20:34:07,932 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=10 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:07,933 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): localhost,100,1 is throwing ServerNotRunningYetException for 1734208447918ms; trying another server 2024-12-14T20:34:07,933 WARN [MockRSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=6, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,100,1 for region state=OPENING, location=localhost,100,1, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 to server localhost,100,1 failed org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:07,934 INFO [PEWorker-17 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=3 2024-12-14T20:34:07,934 INFO [PEWorker-18 {}] assignment.TransitRegionStateProcedure(308): Retry=3 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=localhost,100,1 2024-12-14T20:34:07,934 INFO [PEWorker-17 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=3, state=SUCCESS; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,100,1 in 1.1680 sec 2024-12-14T20:34:07,934 INFO [PEWorker-18 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:08,085 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:08,085 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:08,086 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:08,086 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:08,086 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:08,086 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:08,086 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:08,086 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:08,087 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:08,087 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:08,087 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:08,087 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:08,087 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:08,087 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:08,088 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:08,088 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:08,088 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:08,088 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:08,088 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:08,089 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:08,089 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:08,089 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:08,090 INFO [PEWorker-19 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=7, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,104,1}] 2024-12-14T20:34:08,241 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=0 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:08,242 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,104,1, try=0, can wait up to 1000ms 2024-12-14T20:34:08,343 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=1 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:08,344 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,104,1, try=1, can wait up to 899ms 2024-12-14T20:34:08,438 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-14T20:34:08,439 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-14T20:34:08,445 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=2 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:08,445 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,104,1, try=2, can wait up to 797ms 2024-12-14T20:34:08,546 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=3 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:08,546 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,104,1, try=3, can wait up to 696ms 2024-12-14T20:34:08,647 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=4 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:08,648 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,104,1, try=4, can wait up to 594ms 2024-12-14T20:34:08,748 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=5 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:08,749 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,104,1, try=5, can wait up to 493ms 2024-12-14T20:34:08,850 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=6 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:08,850 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,104,1, try=6, can wait up to 392ms 2024-12-14T20:34:08,951 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=7 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:08,952 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,104,1, try=7, can wait up to 290ms 2024-12-14T20:34:09,053 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=8 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:09,053 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,104,1, try=8, can wait up to 189ms 2024-12-14T20:34:09,153 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=9 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:09,154 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,104,1, try=9, can wait up to 88ms 2024-12-14T20:34:09,255 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,104,1 failed, try=10 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:09,255 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): localhost,104,1 is throwing ServerNotRunningYetException for 1734208449242ms; trying another server 2024-12-14T20:34:09,256 WARN [MockRSProcedureDispatcher-pool-2 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=7, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,104,1 for region state=OPENING, location=localhost,104,1, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 to server localhost,104,1 failed org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:09,257 INFO [PEWorker-21 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=7, resume processing ppid=3 2024-12-14T20:34:09,257 INFO [PEWorker-22 {}] assignment.TransitRegionStateProcedure(308): Retry=4 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=localhost,104,1 2024-12-14T20:34:09,257 INFO [PEWorker-21 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, ppid=3, state=SUCCESS; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,104,1 in 1.1670 sec 2024-12-14T20:34:09,258 WARN [PEWorker-22 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:09,258 INFO [PEWorker-22 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; timeout=10, timestamp=1734208449268 2024-12-14T20:34:09,268 INFO [PEWorker-23 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:09,419 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:09,420 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:09,420 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:09,420 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:09,420 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:09,421 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:09,421 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:09,421 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:09,421 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:09,421 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:09,421 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:09,422 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:09,422 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:09,422 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:09,422 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:09,422 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:09,423 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:09,423 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:09,423 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:09,423 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:09,423 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:09,423 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:09,425 INFO [PEWorker-24 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=8, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,105,1}] 2024-12-14T20:34:09,576 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=0 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:09,576 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,105,1, try=0, can wait up to 1000ms 2024-12-14T20:34:09,677 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=1 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:09,677 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,105,1, try=1, can wait up to 899ms 2024-12-14T20:34:09,778 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=2 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:09,779 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,105,1, try=2, can wait up to 797ms 2024-12-14T20:34:09,880 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=3 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:09,881 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,105,1, try=3, can wait up to 695ms 2024-12-14T20:34:09,982 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=4 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:09,982 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,105,1, try=4, can wait up to 594ms 2024-12-14T20:34:10,083 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=5 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:10,084 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,105,1, try=5, can wait up to 492ms 2024-12-14T20:34:10,185 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=6 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:10,185 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,105,1, try=6, can wait up to 391ms 2024-12-14T20:34:10,286 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=7 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:10,286 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,105,1, try=7, can wait up to 290ms 2024-12-14T20:34:10,387 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=8 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:10,388 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,105,1, try=8, can wait up to 188ms 2024-12-14T20:34:10,489 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=9 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:10,489 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,105,1, try=9, can wait up to 87ms 2024-12-14T20:34:10,589 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=10 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:10,590 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): localhost,105,1 is throwing ServerNotRunningYetException for 1734208450576ms; trying another server 2024-12-14T20:34:10,590 WARN [MockRSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=8, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,105,1 for region state=OPENING, location=localhost,105,1, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 to server localhost,105,1 failed org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:10,592 INFO [PEWorker-26 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=8, resume processing ppid=3 2024-12-14T20:34:10,592 INFO [PEWorker-27 {}] assignment.TransitRegionStateProcedure(308): Retry=5 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=localhost,105,1 2024-12-14T20:34:10,592 INFO [PEWorker-26 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, ppid=3, state=SUCCESS; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,105,1 in 1.1670 sec 2024-12-14T20:34:10,593 WARN [PEWorker-27 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:10,593 INFO [PEWorker-27 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; timeout=20, timestamp=1734208450613 2024-12-14T20:34:10,613 INFO [PEWorker-28 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:10,764 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:10,764 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:10,764 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:10,764 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:10,764 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:10,765 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:10,765 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:10,765 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:10,765 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:10,765 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:10,765 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:10,765 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:10,765 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:10,766 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:10,766 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:10,766 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:10,766 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:10,766 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:10,766 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:10,766 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:10,766 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:10,767 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:10,768 INFO [PEWorker-29 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=9, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,101,1}] 2024-12-14T20:34:10,919 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=0 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:10,920 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,101,1, try=0, can wait up to 1000ms 2024-12-14T20:34:11,020 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=1 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:11,021 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,101,1, try=1, can wait up to 898ms 2024-12-14T20:34:11,121 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=2 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:11,122 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,101,1, try=2, can wait up to 797ms 2024-12-14T20:34:11,223 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=3 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:11,223 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,101,1, try=3, can wait up to 696ms 2024-12-14T20:34:11,323 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=4 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:11,324 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,101,1, try=4, can wait up to 595ms 2024-12-14T20:34:11,424 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=5 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:11,425 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,101,1, try=5, can wait up to 494ms 2024-12-14T20:34:11,525 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=6 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:11,526 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,101,1, try=6, can wait up to 393ms 2024-12-14T20:34:11,627 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=7 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:11,627 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,101,1, try=7, can wait up to 292ms 2024-12-14T20:34:11,727 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=8 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:11,728 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,101,1, try=8, can wait up to 191ms 2024-12-14T20:34:11,829 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=9 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:11,829 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,101,1, try=9, can wait up to 90ms 2024-12-14T20:34:11,930 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=10 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:11,931 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): localhost,101,1 is throwing ServerNotRunningYetException for 1734208451919ms; trying another server 2024-12-14T20:34:11,931 WARN [MockRSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=9, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,101,1 for region state=OPENING, location=localhost,101,1, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 to server localhost,101,1 failed org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:11,932 INFO [PEWorker-31 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=9, resume processing ppid=3 2024-12-14T20:34:11,933 INFO [PEWorker-31 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, ppid=3, state=SUCCESS; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,101,1 in 1.1650 sec 2024-12-14T20:34:11,933 INFO [PEWorker-32 {}] assignment.TransitRegionStateProcedure(308): Retry=6 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=localhost,101,1 2024-12-14T20:34:11,933 WARN [PEWorker-32 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:11,934 INFO [PEWorker-32 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; timeout=40, timestamp=1734208451974 2024-12-14T20:34:11,975 INFO [PEWorker-33 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:12,126 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:12,126 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:12,127 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:12,127 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:12,127 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:12,127 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:12,127 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:12,127 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:12,128 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:12,128 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:12,128 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:12,128 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:12,128 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:12,128 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:12,128 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:12,128 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:12,129 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:12,129 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:12,129 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:12,129 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:12,129 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:12,129 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:12,131 INFO [PEWorker-34 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,108,1}] 2024-12-14T20:34:12,282 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,108,1 failed, try=0 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:12,282 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,108,1, try=0, can wait up to 1000ms 2024-12-14T20:34:12,383 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,108,1 failed, try=1 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:12,383 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,108,1, try=1, can wait up to 899ms 2024-12-14T20:34:12,483 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,108,1 failed, try=2 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:12,483 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,108,1, try=2, can wait up to 799ms 2024-12-14T20:34:12,583 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,108,1 failed, try=3 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:12,584 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,108,1, try=3, can wait up to 698ms 2024-12-14T20:34:12,685 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,108,1 failed, try=4 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:12,685 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,108,1, try=4, can wait up to 597ms 2024-12-14T20:34:12,786 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,108,1 failed, try=5 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:12,786 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,108,1, try=5, can wait up to 496ms 2024-12-14T20:34:12,887 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,108,1 failed, try=6 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:12,888 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,108,1, try=6, can wait up to 395ms 2024-12-14T20:34:12,989 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,108,1 failed, try=7 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:12,989 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,108,1, try=7, can wait up to 293ms 2024-12-14T20:34:13,090 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,108,1 failed, try=8 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:13,090 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,108,1, try=8, can wait up to 192ms 2024-12-14T20:34:13,191 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,108,1 failed, try=9 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:13,192 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,108,1, try=9, can wait up to 90ms 2024-12-14T20:34:13,293 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,108,1 failed, try=10 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:13,293 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): localhost,108,1 is throwing ServerNotRunningYetException for 1734208453282ms; trying another server 2024-12-14T20:34:13,293 WARN [MockRSProcedureDispatcher-pool-2 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=10, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,108,1 for region state=OPENING, location=localhost,108,1, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 to server localhost,108,1 failed org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:13,295 INFO [PEWorker-36 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=3 2024-12-14T20:34:13,295 INFO [PEWorker-37 {}] assignment.TransitRegionStateProcedure(308): Retry=7 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=localhost,108,1 2024-12-14T20:34:13,295 INFO [PEWorker-36 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=3, state=SUCCESS; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,108,1 in 1.1640 sec 2024-12-14T20:34:13,295 WARN [PEWorker-37 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:13,296 INFO [PEWorker-37 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; timeout=80, timestamp=1734208453376 2024-12-14T20:34:13,377 INFO [PEWorker-38 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:13,527 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:13,528 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:13,528 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:13,528 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:13,528 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:13,528 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:13,528 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:13,528 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:13,528 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:13,528 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:13,528 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:13,528 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:13,528 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:13,528 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:13,528 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:13,528 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:13,528 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:13,528 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:13,528 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:13,528 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:13,528 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:13,528 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:13,528 INFO [PEWorker-39 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,103,1}] 2024-12-14T20:34:13,679 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=0 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:13,680 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=0, can wait up to 1000ms 2024-12-14T20:34:13,780 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=1 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:13,781 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=1, can wait up to 899ms 2024-12-14T20:34:13,882 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=2 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:13,882 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=2, can wait up to 798ms 2024-12-14T20:34:13,983 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=3 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:13,984 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=3, can wait up to 696ms 2024-12-14T20:34:14,085 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=4 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:14,085 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=4, can wait up to 595ms 2024-12-14T20:34:14,186 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=5 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:14,186 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=5, can wait up to 494ms 2024-12-14T20:34:14,287 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=6 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:14,288 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=6, can wait up to 392ms 2024-12-14T20:34:14,388 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=7 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:14,389 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=7, can wait up to 291ms 2024-12-14T20:34:14,490 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=8 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:14,490 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=8, can wait up to 190ms 2024-12-14T20:34:14,591 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=9 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:14,592 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,103,1, try=9, can wait up to 88ms 2024-12-14T20:34:14,693 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=10 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:14,694 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): localhost,103,1 is throwing ServerNotRunningYetException for 1734208454680ms; trying another server 2024-12-14T20:34:14,694 WARN [MockRSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=11, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,103,1 for region state=OPENING, location=localhost,103,1, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 to server localhost,103,1 failed org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:14,695 INFO [PEWorker-41 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=3 2024-12-14T20:34:14,695 INFO [PEWorker-42 {}] assignment.TransitRegionStateProcedure(308): Retry=8 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=localhost,103,1 2024-12-14T20:34:14,696 INFO [PEWorker-41 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=3, state=SUCCESS; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,103,1 in 1.1670 sec 2024-12-14T20:34:14,696 WARN [PEWorker-42 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:14,697 INFO [PEWorker-42 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; timeout=100, timestamp=1734208454797 2024-12-14T20:34:14,798 INFO [PEWorker-43 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:14,948 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:14,949 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:14,949 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:14,949 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:14,949 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:14,949 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:14,950 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:14,950 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:14,950 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:14,950 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:14,950 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:14,950 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:14,950 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:14,950 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:14,951 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:14,951 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:14,951 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:14,951 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:14,951 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:14,951 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:14,951 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:14,951 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:14,952 INFO [PEWorker-44 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=12, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,107,1}] 2024-12-14T20:34:15,103 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=0 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:15,104 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=0, can wait up to 1000ms 2024-12-14T20:34:15,205 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=1 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:15,206 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=1, can wait up to 898ms 2024-12-14T20:34:15,307 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=2 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:15,307 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=2, can wait up to 797ms 2024-12-14T20:34:15,408 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=3 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:15,409 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=3, can wait up to 695ms 2024-12-14T20:34:15,510 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=4 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:15,510 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=4, can wait up to 594ms 2024-12-14T20:34:15,611 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=5 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:15,612 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=5, can wait up to 493ms 2024-12-14T20:34:15,712 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=6 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:15,713 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=6, can wait up to 391ms 2024-12-14T20:34:15,814 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=7 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:15,815 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=7, can wait up to 290ms 2024-12-14T20:34:15,916 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=8 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:15,916 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=8, can wait up to 188ms 2024-12-14T20:34:16,017 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=9 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:16,017 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=9, can wait up to 87ms 2024-12-14T20:34:16,118 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=10 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:16,119 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): localhost,107,1 is throwing ServerNotRunningYetException for 1734208456104ms; trying another server 2024-12-14T20:34:16,120 WARN [MockRSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=12, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,107,1 for region state=OPENING, location=localhost,107,1, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 to server localhost,107,1 failed org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:16,121 INFO [PEWorker-46 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=12, resume processing ppid=3 2024-12-14T20:34:16,122 INFO [PEWorker-47 {}] assignment.TransitRegionStateProcedure(308): Retry=9 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=localhost,107,1 2024-12-14T20:34:16,122 INFO [PEWorker-46 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, ppid=3, state=SUCCESS; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,107,1 in 1.1690 sec 2024-12-14T20:34:16,122 WARN [PEWorker-47 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:16,123 INFO [PEWorker-47 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; timeout=100, timestamp=1734208456223 2024-12-14T20:34:16,224 INFO [PEWorker-48 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:16,374 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:16,375 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:16,375 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:16,375 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:16,375 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:16,375 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:16,376 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:16,376 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:16,376 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:16,376 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:16,376 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:16,376 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:16,376 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:16,376 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:16,377 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:16,377 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:16,377 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:16,377 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:16,377 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:16,377 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:16,377 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:16,378 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:16,379 INFO [PEWorker-49 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,107,1}] 2024-12-14T20:34:16,530 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=0 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:16,531 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=0, can wait up to 1000ms 2024-12-14T20:34:16,632 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=1 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:16,632 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=1, can wait up to 899ms 2024-12-14T20:34:16,733 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=2 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:16,733 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=2, can wait up to 798ms 2024-12-14T20:34:16,834 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=3 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:16,835 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=3, can wait up to 697ms 2024-12-14T20:34:16,935 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=4 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:16,936 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=4, can wait up to 595ms 2024-12-14T20:34:17,037 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=5 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:17,037 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=5, can wait up to 494ms 2024-12-14T20:34:17,137 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=6 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:17,138 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=6, can wait up to 393ms 2024-12-14T20:34:17,238 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=7 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:17,239 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=7, can wait up to 293ms 2024-12-14T20:34:17,339 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=8 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:17,340 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=8, can wait up to 191ms 2024-12-14T20:34:17,441 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=9 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:17,441 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(281): Waiting a little before retrying localhost,107,1, try=9, can wait up to 90ms 2024-12-14T20:34:17,542 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=10 org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:17,542 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(288): localhost,107,1 is throwing ServerNotRunningYetException for 1734208457531ms; trying another server 2024-12-14T20:34:17,543 WARN [MockRSProcedureDispatcher-pool-2 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=13, ppid=3, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,107,1 for region state=OPENING, location=localhost,107,1, table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083 to server localhost,107,1 failed org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: wait on server startup at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$ServerNotYetRunningRsExecutor.sendRequest(TestAssignmentManagerBase.java:362) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase$MockRSProcedureDispatcher$MockRemoteCall.sendRequest(TestAssignmentManagerBase.java:676) ~[test-classes/:?] at org.apache.hadoop.hbase.master.procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall.run(RSProcedureDispatcher.java:403) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:17,544 INFO [PEWorker-51 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=3 2024-12-14T20:34:17,544 INFO [PEWorker-52 {}] assignment.TransitRegionStateProcedure(308): Retry=10 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=OPENING, location=localhost,107,1 2024-12-14T20:34:17,545 INFO [PEWorker-51 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=3, state=SUCCESS; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,107,1 in 1.1660 sec 2024-12-14T20:34:17,545 ERROR [PEWorker-52 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=3, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.client.RetriesExhaustedException via TransitRegionStateProcedure:org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:312) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:313) ~[classes/:?] ... 12 more 2024-12-14T20:34:17,547 INFO [PEWorker-52 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=3, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.client.RetriesExhaustedException via TransitRegionStateProcedure:org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN exec-time=13.5910 sec 2024-12-14T20:34:17,547 WARN [PEWorker-52 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-14T20:34:17,594 INFO [Time-limited test {}] assignment.TestAssignmentManagerBase(221): ExecutionException java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait$ProcedureFuture.get(ProcedureSyncWait.java:117) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait$ProcedureFuture.get(ProcedureSyncWait.java:65) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase.waitOnFuture(TestAssignmentManagerBase.java:219) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testRetriesExhaustedFailure(TestAMServerFailedOpen.java:68) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testServerNotYetRunning(TestAMServerFailedOpen.java:54) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:313) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:17,594 INFO [Time-limited test {}] assignment.TestAMServerFailedOpen(72): expected exception from assign operation: Max attempts 10 exceeded org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:313) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:17,595 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1098): Stored pid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN 2024-12-14T20:34:17,595 INFO [PEWorker-53 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN 2024-12-14T20:34:17,596 INFO [PEWorker-53 {}] assignment.TransitRegionStateProcedure(264): Starting pid=14, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN; state=FAILED_OPEN, location=null; forceNewPlan=false, retain=false 2024-12-14T20:34:17,645 DEBUG [Time-limited test {}] procedure.ProcedureSyncWait(219): waitFor pid=14 2024-12-14T20:34:17,746 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:17,747 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:17,747 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:17,747 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:17,747 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:17,747 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:17,747 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:17,748 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:17,748 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:17,748 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:17,748 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:17,748 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:17,748 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:17,748 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:17,748 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:17,749 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:17,749 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:17,749 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:17,749 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:17,749 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:17,749 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:17,749 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:17,751 INFO [PEWorker-54 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,103,1}] 2024-12-14T20:34:17,904 INFO [PEWorker-56 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=15, resume processing ppid=14 2024-12-14T20:34:17,904 INFO [PEWorker-56 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, ppid=14, state=SUCCESS; OpenRegionProcedure 785c39e1e6777b90916b7ee73269d083, server=localhost,103,1 in 153 msec 2024-12-14T20:34:17,905 INFO [PEWorker-57 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, state=SUCCESS; TransitRegionStateProcedure table=testServerNotYetRunning, region=785c39e1e6777b90916b7ee73269d083, ASSIGN in 309 msec 2024-12-14T20:34:17,946 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-14T20:34:17,947 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-14T20:34:17,947 INFO [Time-limited test {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-14T20:34:17,958 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestAMServerFailedOpen#testServerNotYetRunning Thread=27 (was 23) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1734208443537 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/f6cb4b59-76cb-5398-29f9-301fb16c3ab7/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=292 (was 288) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=78 (was 101), ProcessCount=11 (was 11), AvailableMemoryMB=6877 (was 6898) 2024-12-14T20:34:17,963 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.assignment.TestAMServerFailedOpen#testDoNotRetryExceptionOnAssignment Thread=27, OpenFileDescriptor=292, MaxFileDescriptor=1048576, SystemLoadAverage=78, ProcessCount=11, AvailableMemoryMB=6877 2024-12-14T20:34:17,977 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.log.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/hadoop-log-dir so I do NOT create it in target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6 2024-12-14T20:34:17,977 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.log.dir property value differs in configuration and system: Configuration=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/../logs while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/hadoop-log-dir Erasing configuration value by system value. 2024-12-14T20:34:17,977 INFO [Time-limited test {}] hbase.HBaseTestingUtility(451): System.getProperty("hadoop.tmp.dir") already set to: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/hadoop-tmp-dir so I do NOT create it in target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6 2024-12-14T20:34:17,977 WARN [Time-limited test {}] hbase.HBaseTestingUtility(455): hadoop.tmp.dir property value differs in configuration and system: Configuration=/tmp/hadoop-jenkins while System=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/4ef56767-2116-9d08-eb62-75ee52b7eb5a/hadoop-tmp-dir Erasing configuration value by system value. 2024-12-14T20:34:17,977 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6 2024-12-14T20:34:17,986 INFO [Time-limited test {}] util.FSUtils(490): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6 with version=8 2024-12-14T20:34:17,991 DEBUG [Time-limited test {}] util.FSUtils(639): Created cluster ID file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/hbase.id with ID: 8a1ca4c9-f15b-4775-bd79-c7adde25980e 2024-12-14T20:34:18,020 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-14T20:34:18,021 INFO [Time-limited test {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-14T20:34:18,023 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-14T20:34:18,023 INFO [Time-limited test {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-14T20:34:18,043 INFO [Time-limited test {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/MasterData/data/master/store 2024-12-14T20:34:18,050 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-14T20:34:18,050 DEBUG [Time-limited test {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-14T20:34:18,050 INFO [Time-limited test {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:34:18,050 DEBUG [Time-limited test {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:34:18,050 DEBUG [Time-limited test {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-14T20:34:18,050 DEBUG [Time-limited test {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:34:18,050 INFO [Time-limited test {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-14T20:34:18,050 DEBUG [Time-limited test {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-14T20:34:18,050 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/MasterData/data/master/store/.initializing 2024-12-14T20:34:18,050 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/MasterData/WALs/mockmaster.example.org,1234,-1 2024-12-14T20:34:18,054 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-14T20:34:18,057 INFO [Time-limited test {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/MasterData/oldWALs, maxLogs=10 2024-12-14T20:34:18,075 INFO [Time-limited test {}] wal.AbstractFSWAL(841): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1734208458057 2024-12-14T20:34:18,076 DEBUG [Time-limited test {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [] 2024-12-14T20:34:18,076 DEBUG [Time-limited test {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-14T20:34:18,076 DEBUG [Time-limited test {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-14T20:34:18,076 DEBUG [Time-limited test {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:18,076 DEBUG [Time-limited test {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:18,077 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:18,080 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-14T20:34:18,080 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:34:18,080 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-14T20:34:18,080 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:18,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-14T20:34:18,083 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:34:18,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-14T20:34:18,083 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:18,086 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-14T20:34:18,086 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:34:18,086 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-14T20:34:18,087 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:18,090 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-14T20:34:18,090 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-14T20:34:18,091 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-14T20:34:18,092 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:18,092 DEBUG [Time-limited test {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:18,092 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-14T20:34:18,093 DEBUG [Time-limited test {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-14T20:34:18,100 DEBUG [Time-limited test {}] wal.WALSplitUtil(409): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-14T20:34:18,101 INFO [Time-limited test {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66804428, jitterRate=-0.004536449909210205}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-14T20:34:18,101 DEBUG [Time-limited test {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-14T20:34:18,101 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-14T20:34:18,108 INFO [Time-limited test {}] procedure2.ProcedureExecutor(633): Starting 64 core workers (bigger of cpus/4 or 16) with max (burst) worker count=640 2024-12-14T20:34:18,109 INFO [Time-limited test {}] procedure2.ProcedureExecutor(653): Recovered NoopProcedureStore lease in 0 msec 2024-12-14T20:34:18,109 INFO [Time-limited test {}] procedure2.ProcedureExecutor(667): Loaded NoopProcedureStore in 0 msec 2024-12-14T20:34:18,118 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1734208488118 2024-12-14T20:34:18,118 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-14T20:34:18,119 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,100,1 2024-12-14T20:34:18,119 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,101,1 2024-12-14T20:34:18,119 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,102,1 2024-12-14T20:34:18,120 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,103,1 2024-12-14T20:34:18,120 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,104,1 2024-12-14T20:34:18,120 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,105,1 2024-12-14T20:34:18,120 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,106,1 2024-12-14T20:34:18,120 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,107,1 2024-12-14T20:34:18,120 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,108,1 2024-12-14T20:34:18,120 INFO [Time-limited test {}] master.ServerManager(486): Registering regionserver=localhost,109,1 2024-12-14T20:34:18,121 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-14T20:34:18,121 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-14T20:34:18,121 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(264): Starting pid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-14T20:34:18,171 DEBUG [Time-limited test {}] procedure.ProcedureSyncWait(219): waitFor pid=1 2024-12-14T20:34:18,272 DEBUG [mockmaster:1234 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=10, allServersCount=10 2024-12-14T20:34:18,272 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:18,272 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:18,272 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:18,272 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:18,272 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:18,272 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:18,273 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:18,273 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:18,273 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:18,273 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:18,273 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:18,273 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:18,273 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:18,273 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:18,273 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:18,273 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:18,274 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:18,274 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:18,274 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:18,274 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:18,274 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:18,274 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:18,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE; OpenRegionProcedure 1588230740, server=localhost,102,1}] 2024-12-14T20:34:18,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-14T20:34:18,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; OpenRegionProcedure 1588230740, server=localhost,102,1 in 152 msec 2024-12-14T20:34:18,428 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 307 msec 2024-12-14T20:34:18,473 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1098): Stored pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN 2024-12-14T20:34:18,474 INFO [PEWorker-6 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN 2024-12-14T20:34:18,474 INFO [PEWorker-6 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-14T20:34:18,523 DEBUG [Time-limited test {}] procedure.ProcedureSyncWait(219): waitFor pid=3 2024-12-14T20:34:18,625 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:18,625 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:18,625 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:18,625 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:18,626 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:18,626 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:18,626 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:18,626 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:18,626 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:18,626 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:18,626 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:18,626 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:18,627 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:18,627 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:18,627 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:18,627 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:18,627 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:18,627 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:18,627 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:18,627 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:18,628 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:18,628 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:18,629 INFO [PEWorker-7 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=4, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,101,1}] 2024-12-14T20:34:18,780 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=0 org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:18,781 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(293): localhost,101,1 tells us DoNotRetry due to org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault, try=0, give up 2024-12-14T20:34:18,781 WARN [MockRSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=4, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,101,1 for region state=OPENING, location=localhost,101,1, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 to server localhost,101,1 failed org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:18,783 INFO [PEWorker-9 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=4, resume processing ppid=3 2024-12-14T20:34:18,783 INFO [PEWorker-9 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, ppid=3, state=SUCCESS; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,101,1 in 153 msec 2024-12-14T20:34:18,783 INFO [PEWorker-10 {}] assignment.TransitRegionStateProcedure(308): Retry=1 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=localhost,101,1 2024-12-14T20:34:18,783 INFO [PEWorker-10 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:18,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:18,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:18,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:18,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:18,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:18,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:18,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:18,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:18,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:18,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:18,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:18,935 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:18,935 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:18,935 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:18,935 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:18,935 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:18,935 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:18,935 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:18,935 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:18,935 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:18,935 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:18,935 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:18,936 INFO [PEWorker-11 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,103,1}] 2024-12-14T20:34:19,087 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,103,1 failed, try=0 org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:19,089 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(293): localhost,103,1 tells us DoNotRetry due to org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault, try=0, give up 2024-12-14T20:34:19,089 WARN [MockRSProcedureDispatcher-pool-2 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=5, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,103,1 for region state=OPENING, location=localhost,103,1, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 to server localhost,103,1 failed org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:19,090 INFO [PEWorker-13 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=3 2024-12-14T20:34:19,090 INFO [PEWorker-13 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=3, state=SUCCESS; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,103,1 in 155 msec 2024-12-14T20:34:19,090 INFO [PEWorker-14 {}] assignment.TransitRegionStateProcedure(308): Retry=2 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=localhost,103,1 2024-12-14T20:34:19,091 INFO [PEWorker-14 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:19,241 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:19,241 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:19,241 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:19,242 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:19,242 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:19,242 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:19,242 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:19,242 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:19,242 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:19,242 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:19,242 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:19,242 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:19,242 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:19,242 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:19,242 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:19,242 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:19,242 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:19,243 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:19,243 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:19,243 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:19,243 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:19,243 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:19,244 INFO [PEWorker-15 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,102,1}] 2024-12-14T20:34:19,395 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,102,1 failed, try=0 org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:19,396 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(293): localhost,102,1 tells us DoNotRetry due to org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault, try=0, give up 2024-12-14T20:34:19,396 WARN [MockRSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=6, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,102,1 for region state=OPENING, location=localhost,102,1, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 to server localhost,102,1 failed org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:19,398 INFO [PEWorker-17 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=3 2024-12-14T20:34:19,398 INFO [PEWorker-17 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=3, state=SUCCESS; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,102,1 in 153 msec 2024-12-14T20:34:19,398 INFO [PEWorker-18 {}] assignment.TransitRegionStateProcedure(308): Retry=3 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=localhost,102,1 2024-12-14T20:34:19,398 INFO [PEWorker-18 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:19,549 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:19,549 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:19,550 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:19,550 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:19,550 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:19,550 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:19,550 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:19,550 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:19,550 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:19,551 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:19,551 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:19,551 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:19,551 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:19,551 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:19,551 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:19,551 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:19,551 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:19,552 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:19,552 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:19,552 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:19,552 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:19,552 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:19,553 INFO [PEWorker-19 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=7, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,107,1}] 2024-12-14T20:34:19,704 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,107,1 failed, try=0 org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:19,705 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(293): localhost,107,1 tells us DoNotRetry due to org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault, try=0, give up 2024-12-14T20:34:19,705 WARN [MockRSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=7, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,107,1 for region state=OPENING, location=localhost,107,1, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 to server localhost,107,1 failed org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:19,706 INFO [PEWorker-21 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=7, resume processing ppid=3 2024-12-14T20:34:19,706 INFO [PEWorker-21 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, ppid=3, state=SUCCESS; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,107,1 in 153 msec 2024-12-14T20:34:19,706 INFO [PEWorker-22 {}] assignment.TransitRegionStateProcedure(308): Retry=4 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=localhost,107,1 2024-12-14T20:34:19,706 WARN [PEWorker-22 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:19,707 INFO [PEWorker-22 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; timeout=10, timestamp=1734208459717 2024-12-14T20:34:19,717 INFO [PEWorker-23 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:19,868 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:19,868 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:19,868 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:19,868 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:19,869 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:19,869 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:19,869 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:19,869 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:19,869 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:19,869 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:19,869 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:19,869 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:19,869 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:19,869 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:19,870 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:19,870 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:19,870 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:19,870 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:19,870 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:19,870 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:19,870 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:19,870 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:19,871 INFO [PEWorker-24 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=8, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,105,1}] 2024-12-14T20:34:20,022 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,105,1 failed, try=0 org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:20,023 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(293): localhost,105,1 tells us DoNotRetry due to org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault, try=0, give up 2024-12-14T20:34:20,023 WARN [MockRSProcedureDispatcher-pool-2 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=8, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,105,1 for region state=OPENING, location=localhost,105,1, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 to server localhost,105,1 failed org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:20,025 INFO [PEWorker-26 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=8, resume processing ppid=3 2024-12-14T20:34:20,025 INFO [PEWorker-27 {}] assignment.TransitRegionStateProcedure(308): Retry=5 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=localhost,105,1 2024-12-14T20:34:20,025 INFO [PEWorker-26 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, ppid=3, state=SUCCESS; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,105,1 in 153 msec 2024-12-14T20:34:20,025 WARN [PEWorker-27 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:20,026 INFO [PEWorker-27 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; timeout=20, timestamp=1734208460046 2024-12-14T20:34:20,046 INFO [PEWorker-28 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:20,197 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:20,197 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:20,197 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:20,198 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:20,198 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:20,198 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:20,198 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:20,198 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:20,198 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:20,198 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:20,198 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:20,198 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:20,198 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:20,198 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:20,199 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:20,199 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:20,199 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:20,199 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:20,199 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:20,199 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:20,199 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:20,199 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:20,200 INFO [PEWorker-29 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=9, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,102,1}] 2024-12-14T20:34:20,351 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,102,1 failed, try=0 org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:20,351 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(293): localhost,102,1 tells us DoNotRetry due to org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault, try=0, give up 2024-12-14T20:34:20,352 WARN [MockRSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=9, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,102,1 for region state=OPENING, location=localhost,102,1, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 to server localhost,102,1 failed org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:20,353 INFO [PEWorker-31 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=9, resume processing ppid=3 2024-12-14T20:34:20,353 INFO [PEWorker-31 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, ppid=3, state=SUCCESS; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,102,1 in 153 msec 2024-12-14T20:34:20,353 INFO [PEWorker-32 {}] assignment.TransitRegionStateProcedure(308): Retry=6 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=localhost,102,1 2024-12-14T20:34:20,353 WARN [PEWorker-32 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:20,354 INFO [PEWorker-32 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; timeout=40, timestamp=1734208460394 2024-12-14T20:34:20,395 INFO [PEWorker-33 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:20,545 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:20,546 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:20,546 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:20,546 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:20,546 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:20,546 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:20,547 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:20,547 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:20,547 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:20,547 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:20,547 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:20,547 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:20,547 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:20,548 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:20,548 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:20,548 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:20,548 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:20,548 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:20,548 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:20,548 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:20,549 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:20,549 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:20,550 INFO [PEWorker-34 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,109,1}] 2024-12-14T20:34:20,701 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,109,1 failed, try=0 org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:20,701 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(293): localhost,109,1 tells us DoNotRetry due to org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault, try=0, give up 2024-12-14T20:34:20,701 WARN [MockRSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=10, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,109,1 for region state=OPENING, location=localhost,109,1, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 to server localhost,109,1 failed org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:20,702 INFO [PEWorker-36 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=3 2024-12-14T20:34:20,703 INFO [PEWorker-36 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=3, state=SUCCESS; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,109,1 in 153 msec 2024-12-14T20:34:20,702 INFO [PEWorker-37 {}] assignment.TransitRegionStateProcedure(308): Retry=7 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=localhost,109,1 2024-12-14T20:34:20,703 WARN [PEWorker-37 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:20,703 INFO [PEWorker-37 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; timeout=80, timestamp=1734208460783 2024-12-14T20:34:20,784 INFO [PEWorker-38 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:20,934 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:20,935 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:20,935 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:20,935 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:20,935 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:20,935 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:20,936 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:20,936 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:20,936 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:20,936 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:20,936 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:20,936 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:20,936 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:20,936 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:20,937 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:20,937 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:20,937 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:20,937 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:20,937 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:20,937 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:20,937 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:20,937 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:20,938 INFO [PEWorker-39 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=11, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,101,1}] 2024-12-14T20:34:21,089 DEBUG [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,101,1 failed, try=0 org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:21,090 WARN [MockRSProcedureDispatcher-pool-2 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(293): localhost,101,1 tells us DoNotRetry due to org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault, try=0, give up 2024-12-14T20:34:21,090 WARN [MockRSProcedureDispatcher-pool-2 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=11, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,101,1 for region state=OPENING, location=localhost,101,1, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 to server localhost,101,1 failed org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:21,092 INFO [PEWorker-41 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=11, resume processing ppid=3 2024-12-14T20:34:21,092 INFO [PEWorker-42 {}] assignment.TransitRegionStateProcedure(308): Retry=8 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=localhost,101,1 2024-12-14T20:34:21,092 INFO [PEWorker-41 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, ppid=3, state=SUCCESS; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,101,1 in 153 msec 2024-12-14T20:34:21,092 WARN [PEWorker-42 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:21,093 INFO [PEWorker-42 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; timeout=100, timestamp=1734208461193 2024-12-14T20:34:21,194 INFO [PEWorker-43 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:21,344 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:21,345 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:21,345 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:21,345 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:21,345 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:21,345 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:21,345 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:21,346 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:21,346 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:21,346 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:21,346 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:21,346 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:21,346 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:21,346 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:21,346 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:21,347 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:21,347 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:21,347 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:21,347 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:21,347 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:21,347 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:21,347 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:21,348 INFO [PEWorker-44 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=12, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,100,1}] 2024-12-14T20:34:21,500 DEBUG [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,100,1 failed, try=0 org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:21,501 WARN [MockRSProcedureDispatcher-pool-0 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(293): localhost,100,1 tells us DoNotRetry due to org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault, try=0, give up 2024-12-14T20:34:21,501 WARN [MockRSProcedureDispatcher-pool-0 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=12, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,100,1 for region state=OPENING, location=localhost,100,1, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 to server localhost,100,1 failed org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:21,502 INFO [PEWorker-46 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=12, resume processing ppid=3 2024-12-14T20:34:21,503 INFO [PEWorker-47 {}] assignment.TransitRegionStateProcedure(308): Retry=9 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=localhost,100,1 2024-12-14T20:34:21,503 INFO [PEWorker-46 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, ppid=3, state=SUCCESS; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,100,1 in 154 msec 2024-12-14T20:34:21,503 WARN [PEWorker-47 {}] assignment.TransitRegionStateProcedure(457): Failed transition, suspend 0secs pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; waiting on rectified condition fixed by other Procedure or operator intervention org.apache.hadoop.hbase.HBaseIOException: Failed confirm OPEN of state=OPENING, location=null, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 (remote log may yield more detail on why). at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:326) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:21,504 INFO [PEWorker-47 {}] procedure2.TimeoutExecutorThread(81): ADDED pid=3, state=WAITING_TIMEOUT:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; timeout=100, timestamp=1734208461604 2024-12-14T20:34:21,605 INFO [PEWorker-48 {}] assignment.TransitRegionStateProcedure(264): Starting pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=null; forceNewPlan=true, retain=false 2024-12-14T20:34:21,756 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(202): Hosts are {localhost=0} racks are {Unknown Rack=0} 2024-12-14T20:34:21,756 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 0 is on host 0 2024-12-14T20:34:21,756 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 1 is on host 0 2024-12-14T20:34:21,756 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 2 is on host 0 2024-12-14T20:34:21,756 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 3 is on host 0 2024-12-14T20:34:21,757 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 4 is on host 0 2024-12-14T20:34:21,757 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 5 is on host 0 2024-12-14T20:34:21,757 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 6 is on host 0 2024-12-14T20:34:21,757 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 7 is on host 0 2024-12-14T20:34:21,757 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 8 is on host 0 2024-12-14T20:34:21,757 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(303): server 9 is on host 0 2024-12-14T20:34:21,757 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 0 is on rack 0 2024-12-14T20:34:21,757 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 1 is on rack 0 2024-12-14T20:34:21,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 2 is on rack 0 2024-12-14T20:34:21,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 3 is on rack 0 2024-12-14T20:34:21,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 4 is on rack 0 2024-12-14T20:34:21,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 5 is on rack 0 2024-12-14T20:34:21,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 6 is on rack 0 2024-12-14T20:34:21,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 7 is on rack 0 2024-12-14T20:34:21,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 8 is on rack 0 2024-12-14T20:34:21,758 INFO [mockmaster:1234 {}] balancer.BalancerClusterState(314): server 9 is on rack 0 2024-12-14T20:34:21,759 DEBUG [mockmaster:1234 {}] balancer.BalancerClusterState(319): Number of tables=1, number of hosts=1, number of racks=1 2024-12-14T20:34:21,760 INFO [PEWorker-49 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=13, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,106,1}] 2024-12-14T20:34:21,911 DEBUG [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(276): Request to localhost,106,1 failed, try=0 org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:21,912 WARN [MockRSProcedureDispatcher-pool-1 {}] procedure.RSProcedureDispatcher$ExecuteProceduresRemoteCall(293): localhost,106,1 tells us DoNotRetry due to org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault, try=0, give up 2024-12-14T20:34:21,912 WARN [MockRSProcedureDispatcher-pool-1 {}] assignment.RegionRemoteProcedureBase(135): The remote operation pid=13, ppid=3, state=RUNNABLE; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,106,1 for region state=OPENING, location=localhost,106,1, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 to server localhost,106,1 failed org.apache.hadoop.hbase.DoNotRetryIOException: test do not retry fault at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-14T20:34:21,913 INFO [PEWorker-51 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=13, resume processing ppid=3 2024-12-14T20:34:21,913 INFO [PEWorker-51 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, ppid=3, state=SUCCESS; OpenRegionProcedure d384f9184f9f3b8439b85bb334b7d157, server=localhost,106,1 in 154 msec 2024-12-14T20:34:21,913 INFO [PEWorker-52 {}] assignment.TransitRegionStateProcedure(308): Retry=10 of max=10; pid=3, state=RUNNABLE:REGION_STATE_TRANSITION_CONFIRM_OPENED, locked=true; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN; state=OPENING, location=localhost,106,1 2024-12-14T20:34:21,913 ERROR [PEWorker-52 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=3, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.client.RetriesExhaustedException via TransitRegionStateProcedure:org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:312) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:313) ~[classes/:?] ... 12 more 2024-12-14T20:34:21,915 INFO [PEWorker-52 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=3, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.client.RetriesExhaustedException via TransitRegionStateProcedure:org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded; TransitRegionStateProcedure table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157, ASSIGN exec-time=3.4410 sec 2024-12-14T20:34:21,915 WARN [PEWorker-52 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-14T20:34:21,932 INFO [Time-limited test {}] assignment.TestAssignmentManagerBase(221): ExecutionException java.util.concurrent.ExecutionException: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait$ProcedureFuture.get(ProcedureSyncWait.java:117) ~[classes/:?] at org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait$ProcedureFuture.get(ProcedureSyncWait.java:65) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAssignmentManagerBase.waitOnFuture(TestAssignmentManagerBase.java:219) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testFailedOpen(TestAMServerFailedOpen.java:99) ~[test-classes/:?] at org.apache.hadoop.hbase.master.assignment.TestAMServerFailedOpen.testDoNotRetryExceptionOnAssignment(TestAMServerFailedOpen.java:85) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:313) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:21,933 INFO [Time-limited test {}] assignment.TestAMServerFailedOpen(103): REGION STATE state=FAILED_OPEN, location=null, table=testDoNotRetryExceptionOnAssignment, region=d384f9184f9f3b8439b85bb334b7d157 2024-12-14T20:34:21,933 INFO [Time-limited test {}] assignment.TestAMServerFailedOpen(104): expected exception from assign operation: Max attempts 10 exceeded org.apache.hadoop.hbase.client.RetriesExhaustedException: Max attempts 10 exceeded at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.confirmOpened(TransitRegionStateProcedure.java:313) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:443) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.executeFromState(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:404) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure.execute(TransitRegionStateProcedure.java:113) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-14T20:34:21,934 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-14T20:34:21,934 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-14T20:34:21,935 INFO [Time-limited test {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-14T20:34:21,944 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.assignment.TestAMServerFailedOpen#testDoNotRetryExceptionOnAssignment Thread=31 (was 27) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1734208458057 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2@2/hbase-server/target/test-data/768eb77e-a850-b057-68dd-2c0659510bf6/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=296 (was 292) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=78 (was 78), ProcessCount=11 (was 11), AvailableMemoryMB=6862 (was 6877)