2024-11-10 05:13:31,619 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-10 05:13:31,636 main DEBUG Took 0.014714 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-10 05:13:31,637 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-10 05:13:31,637 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-10 05:13:31,638 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-10 05:13:31,639 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,671 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-10 05:13:31,697 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,699 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,700 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,700 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,701 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,701 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,702 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,703 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,703 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,703 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,704 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,705 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,706 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,706 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,707 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,707 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,708 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,708 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,709 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,710 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,710 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,711 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,712 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-10 05:13:31,712 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,713 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-10 05:13:31,715 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-10 05:13:31,717 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-10 05:13:31,723 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-10 05:13:31,723 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-10 05:13:31,725 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-10 05:13:31,726 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-10 05:13:31,738 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-10 05:13:31,742 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-10 05:13:31,745 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-10 05:13:31,746 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-10 05:13:31,746 main DEBUG createAppenders(={Console}) 2024-11-10 05:13:31,748 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca initialized 2024-11-10 05:13:31,749 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca 2024-11-10 05:13:31,749 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@30f842ca OK. 2024-11-10 05:13:31,750 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-10 05:13:31,751 main DEBUG OutputStream closed 2024-11-10 05:13:31,751 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-10 05:13:31,752 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-10 05:13:31,752 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@6404f418 OK 2024-11-10 05:13:31,880 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-10 05:13:31,884 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-10 05:13:31,885 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-10 05:13:31,886 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-10 05:13:31,887 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-10 05:13:31,888 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-10 05:13:31,888 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-10 05:13:31,888 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-10 05:13:31,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-10 05:13:31,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-10 05:13:31,889 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-10 05:13:31,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-10 05:13:31,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-10 05:13:31,890 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-10 05:13:31,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-10 05:13:31,891 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-10 05:13:31,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-10 05:13:31,892 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-10 05:13:31,895 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10 05:13:31,896 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@52c3cb31) with optional ClassLoader: null 2024-11-10 05:13:31,897 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-10 05:13:31,898 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@52c3cb31] started OK. 2024-11-10T05:13:31,914 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.master.janitor.TestCatalogJanitor timeout: 13 mins 2024-11-10 05:13:31,919 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-10 05:13:31,920 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-10T05:13:32,379 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f 2024-11-10T05:13:32,421 INFO [Time-limited test {}] regionserver.ChunkCreator(472): data poolSizePercentage is less than 0. So not using pool 2024-11-10T05:13:32,422 INFO [Time-limited test {}] regionserver.ChunkCreator(472): index poolSizePercentage is less than 0. So not using pool 2024-11-10T05:13:32,463 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testCleanParent Thread=12, OpenFileDescriptor=287, MaxFileDescriptor=1048576, SystemLoadAverage=1070, ProcessCount=11, AvailableMemoryMB=3237 2024-11-10T05:13:32,675 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-10T05:13:34,442 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-10T05:13:34,485 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-10T05:13:34,604 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent with version=8 2024-11-10T05:13:34,624 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/hbase.id] with ID: 07200000-fcc9-4529-a94f-d63138a0e0e3 2024-11-10T05:13:34,625 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/.tmp/hbase.id 2024-11-10T05:13:34,633 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/hbase.id] 2024-11-10T05:13:34,676 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:13:34,704 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:13:34,707 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:13:34,806 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.MockMasterServices.(MockMasterServices.java:119) ~[test-classes/:?] at org.apache.hadoop.hbase.master.janitor.TestCatalogJanitor.setup(TestCatalogJanitor.java:110) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.invokeMethod(RunBefores.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:24) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T05:13:34,812 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:13:34,816 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-10T05:13:34,833 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-10T05:13:34,833 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-10T05:13:34,951 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-10T05:13:34,957 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/MasterData/data/master/store 2024-11-10T05:13:34,988 INFO [Time-limited test {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-10T05:13:34,993 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:35,045 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:13:35,046 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:35,046 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:35,049 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:13:35,049 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:35,049 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:35,051 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731215615045Disabling compacts and flushes for region at 1731215615045Disabling writes for close at 1731215615049 (+4 ms)Writing region close event to WAL at 1731215615049Closed at 1731215615049 2024-11-10T05:13:35,052 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/MasterData/data/master/store/.initializing 2024-11-10T05:13:35,053 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T05:13:35,068 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:13:35,093 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/MasterData/oldWALs, maxLogs=10 2024-11-10T05:13:35,211 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215615148 2024-11-10T05:13:35,214 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:13:35,215 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:13:35,217 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:35,222 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,225 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,298 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,348 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:13:35,355 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:35,362 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:13:35,367 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:13:35,379 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:35,380 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:35,381 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,395 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:13:35,395 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:35,396 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:35,397 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,410 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:13:35,410 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:35,411 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:35,412 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,421 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,422 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,426 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,428 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,433 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:13:35,437 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:35,457 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:13:35,459 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58985571, jitterRate=-0.12104649841785431}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:13:35,471 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731215615252Initializing all the Stores at 1731215615253 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731215615254 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215615259 (+5 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215615259Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215615260 (+1 ms)Cleaning up temporary data from old regions at 1731215615428 (+168 ms)Region opened successfully at 1731215615470 (+42 ms) 2024-11-10T05:13:35,473 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:13:35,718 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:13:35,722 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T05:13:35,723 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T05:13:35,724 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731215645724 2024-11-10T05:13:35,725 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:13:35,757 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T05:13:35,759 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T05:13:35,760 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T05:13:35,760 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T05:13:35,761 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T05:13:35,761 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T05:13:35,762 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T05:13:35,762 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T05:13:35,763 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T05:13:35,763 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T05:13:35,772 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:35,798 INFO [Time-limited test {}] janitor.TestCatalogJanitor(270): Created reference file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/data/default/testCleanParent/5c099a6bb1a36eadc9f433730234cbd7/cf/1731215615772.5c099a6bb1a36eadc9f433730234cbd7 2024-11-10T05:13:35,798 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 5c099a6bb1a36eadc9f433730234cbd7, NAME => 'testCleanParent,aaa,1731215615766.5c099a6bb1a36eadc9f433730234cbd7.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:35,808 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:35,813 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => 5c099a6bb1a36eadc9f433730234cbd7, NAME => 'testCleanParent,aaa,1731215615766.5c099a6bb1a36eadc9f433730234cbd7.', STARTKEY => 'aaa', ENDKEY => 'eee'} because daughter {ENCODED => 5c099a6bb1a36eadc9f433730234cbd7, NAME => 'testCleanParent,aaa,1731215615766.5c099a6bb1a36eadc9f433730234cbd7.', STARTKEY => 'aaa', ENDKEY => 'ccc'} still has references 2024-11-10T05:13:35,814 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 5c099a6bb1a36eadc9f433730234cbd7, NAME => 'testCleanParent,aaa,1731215615766.5c099a6bb1a36eadc9f433730234cbd7.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:35,815 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:35,816 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 5c099a6bb1a36eadc9f433730234cbd7 because daughters -- 5c099a6bb1a36eadc9f433730234cbd7, f6c0dc789d6d0dcd2d44135dc2f30a06 -- no longer hold references 2024-11-10T05:13:35,817 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testCleanParent, region=5c099a6bb1a36eadc9f433730234cbd7 2024-11-10T05:13:35,820 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testCleanParent, region=5c099a6bb1a36eadc9f433730234cbd7 for split parent {ENCODED => 5c099a6bb1a36eadc9f433730234cbd7, NAME => 'testCleanParent,aaa,1731215615766.5c099a6bb1a36eadc9f433730234cbd7.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:35,822 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testCleanParent, region=5c099a6bb1a36eadc9f433730234cbd7 2024-11-10T05:13:35,824 DEBUG [PEWorker-1 {}] assignment.GCRegionProcedure(87): Archiving region=5c099a6bb1a36eadc9f433730234cbd7 2024-11-10T05:13:35,825 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/data/default/testCleanParent/5c099a6bb1a36eadc9f433730234cbd7 2024-11-10T05:13:35,827 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/data/default/testCleanParent/5c099a6bb1a36eadc9f433730234cbd7/cf] 2024-11-10T05:13:35,847 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/data/default/testCleanParent/5c099a6bb1a36eadc9f433730234cbd7 2024-11-10T05:13:35,861 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testCleanParent,aaa,1731215615766.5c099a6bb1a36eadc9f433730234cbd7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731215615849"}]},"ts":"9223372036854775807"} 2024-11-10T05:13:35,862 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T05:13:35,862 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5c099a6bb1a36eadc9f433730234cbd7, NAME => 'testCleanParent,aaa,1731215615766.5c099a6bb1a36eadc9f433730234cbd7.', STARTKEY => 'aaa', ENDKEY => 'eee'}] 2024-11-10T05:13:35,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; GCRegionProcedure table=testCleanParent, region=5c099a6bb1a36eadc9f433730234cbd7 in 45 msec 2024-11-10T05:13:36,071 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:13:36,072 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:13:36,072 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:13:36,080 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testCleanParent Thread=24 (was 12) Potentially hanging thread: HBase-Metrics2-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215615148 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanParent/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Monitor thread for TaskMonitor java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master:store-WAL-Roller java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:180) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: weak-ref-cleaner-strictcontextstorage java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//io.opentelemetry.context.StrictContextStorage$PendingScopes.run(StrictContextStorage.java:269) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: master:store-Flusher java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759) app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor.flushLoop(MasterRegionFlusherAndCompactor.java:200) app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor$$Lambda$299/0x00007fa10c4f8448.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1724) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=321 (was 287) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1024 (was 1070), ProcessCount=11 (was 11), AvailableMemoryMB=3053 (was 3237) 2024-11-10T05:13:36,086 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testCleanMerge Thread=24, OpenFileDescriptor=321, MaxFileDescriptor=1048576, SystemLoadAverage=1024, ProcessCount=11, AvailableMemoryMB=3053 2024-11-10T05:13:36,099 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge with version=8 2024-11-10T05:13:36,100 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/hbase.id] with ID: 7c561f40-1ef9-4419-8ceb-1ab18c913739 2024-11-10T05:13:36,100 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/.tmp/hbase.id 2024-11-10T05:13:36,106 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/hbase.id] 2024-11-10T05:13:36,138 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:13:36,139 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:13:36,142 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:13:36,143 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:13:36,168 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/MasterData/data/master/store 2024-11-10T05:13:36,180 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:36,181 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:13:36,181 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:36,182 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:36,182 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:13:36,182 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:36,183 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:36,183 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731215616181Disabling compacts and flushes for region at 1731215616181Disabling writes for close at 1731215616182 (+1 ms)Writing region close event to WAL at 1731215616182Closed at 1731215616182 2024-11-10T05:13:36,184 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/MasterData/data/master/store/.initializing 2024-11-10T05:13:36,184 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T05:13:36,191 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:13:36,196 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/MasterData/oldWALs, maxLogs=10 2024-11-10T05:13:36,218 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215616197 2024-11-10T05:13:36,219 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:13:36,219 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:13:36,219 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:36,220 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,220 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,225 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:13:36,225 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:13:36,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,230 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:13:36,230 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,231 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:36,231 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,234 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:13:36,235 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,235 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:36,235 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:13:36,239 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,239 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:36,240 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,241 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,241 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,242 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,242 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,242 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:13:36,243 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,251 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:13:36,252 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71562480, jitterRate=0.06636404991149902}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:13:36,252 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731215616221Initializing all the Stores at 1731215616221Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731215616221Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215616221Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215616221Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215616222 (+1 ms)Cleaning up temporary data from old regions at 1731215616242 (+20 ms)Region opened successfully at 1731215616252 (+10 ms) 2024-11-10T05:13:36,252 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:13:36,257 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:13:36,258 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T05:13:36,258 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T05:13:36,259 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731215646259 2024-11-10T05:13:36,259 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:13:36,260 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T05:13:36,260 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T05:13:36,260 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T05:13:36,261 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T05:13:36,261 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T05:13:36,261 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T05:13:36,261 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T05:13:36,261 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T05:13:36,262 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T05:13:36,262 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T05:13:36,263 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,280 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,285 DEBUG [Time-limited test {}] janitor.CatalogJanitor(258): Cleaning merged region {ENCODED => 723d42c26e868b0fdc6394437270af3b, NAME => 'testCleanMerge,aaa,1731215616262.723d42c26e868b0fdc6394437270af3b.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:36,286 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,287 DEBUG [Time-limited test {}] janitor.CatalogJanitor(283): Deferring cleanup up of 2 parents of merged region 723d42c26e868b0fdc6394437270af3b, because references still exist in merged region or we encountered an exception in checking 2024-11-10T05:13:36,287 DEBUG [Time-limited test {}] janitor.CatalogJanitor(258): Cleaning merged region {ENCODED => 723d42c26e868b0fdc6394437270af3b, NAME => 'testCleanMerge,aaa,1731215616262.723d42c26e868b0fdc6394437270af3b.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:36,288 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,288 DEBUG [Time-limited test {}] janitor.CatalogJanitor(283): Deferring cleanup up of 2 parents of merged region 723d42c26e868b0fdc6394437270af3b, because references still exist in merged region or we encountered an exception in checking 2024-11-10T05:13:36,289 DEBUG [Time-limited test {}] janitor.CatalogJanitor(258): Cleaning merged region {ENCODED => 723d42c26e868b0fdc6394437270af3b, NAME => 'testCleanMerge,aaa,1731215616262.723d42c26e868b0fdc6394437270af3b.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:36,289 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,290 DEBUG [Time-limited test {}] janitor.CatalogJanitor(266): Deleting parents ([723d42c26e868b0fdc6394437270af3b], [13c2b3011fe815032da3b2ae64bd9ecb]) from fs; merged child {ENCODED => 723d42c26e868b0fdc6394437270af3b, NAME => 'testCleanMerge,aaa,1731215616262.723d42c26e868b0fdc6394437270af3b.', STARTKEY => 'aaa', ENDKEY => 'eee'} no longer holds references 2024-11-10T05:13:36,291 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=false; GCMultipleMergedRegionsProcedure child=723d42c26e868b0fdc6394437270af3b, parents:[723d42c26e868b0fdc6394437270af3b], [13c2b3011fe815032da3b2ae64bd9ecb] 2024-11-10T05:13:36,292 DEBUG [Time-limited test {}] janitor.CatalogJanitor(277): Submitted procedure pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=false; GCMultipleMergedRegionsProcedure child=723d42c26e868b0fdc6394437270af3b, parents:[723d42c26e868b0fdc6394437270af3b], [13c2b3011fe815032da3b2ae64bd9ecb] for merged region {ENCODED => 723d42c26e868b0fdc6394437270af3b, NAME => 'testCleanMerge,aaa,1731215616262.723d42c26e868b0fdc6394437270af3b.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:36,292 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:13:36,292 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:13:36,292 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=false; GCMultipleMergedRegionsProcedure child=723d42c26e868b0fdc6394437270af3b, parents:[723d42c26e868b0fdc6394437270af3b], [13c2b3011fe815032da3b2ae64bd9ecb] 2024-11-10T05:13:36,293 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:13:36,295 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1844): CODE-BUG: Uncaught runtime exception: pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=true; GCMultipleMergedRegionsProcedure child=723d42c26e868b0fdc6394437270af3b, parents:[723d42c26e868b0fdc6394437270af3b], [13c2b3011fe815032da3b2ae64bd9ecb] java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.client.Result.rawCells()" because the return value of "org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(org.apache.hadoop.hbase.client.RegionInfo)" is null at org.apache.hadoop.hbase.master.assignment.RegionStateStore.getMergeRegions(RegionStateStore.java:474) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:103) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-11-10T05:13:36,295 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=1, state=FAILED:GC_MERGED_REGIONS_PREPARE, hasLock=true, exception=java.lang.NullPointerException via CODE-BUG: Uncaught runtime exception: pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=true; GCMultipleMergedRegionsProcedure child=723d42c26e868b0fdc6394437270af3b, parents:[723d42c26e868b0fdc6394437270af3b], [13c2b3011fe815032da3b2ae64bd9ecb]:java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.client.Result.rawCells()" because the return value of "org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(org.apache.hadoop.hbase.client.RegionInfo)" is null; GCMultipleMergedRegionsProcedure child=723d42c26e868b0fdc6394437270af3b, parents:[723d42c26e868b0fdc6394437270af3b], [13c2b3011fe815032da3b2ae64bd9ecb] does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.client.Result.rawCells()" because the return value of "org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(org.apache.hadoop.hbase.client.RegionInfo)" is null at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1845) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.client.Result.rawCells()" because the return value of "org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(org.apache.hadoop.hbase.client.RegionInfo)" is null at org.apache.hadoop.hbase.master.assignment.RegionStateStore.getMergeRegions(RegionStateStore.java:474) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:103) ~[classes/:?] at org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 4 more 2024-11-10T05:13:36,296 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=1, state=ROLLEDBACK, hasLock=true, exception=java.lang.NullPointerException via CODE-BUG: Uncaught runtime exception: pid=1, state=RUNNABLE:GC_MERGED_REGIONS_PREPARE, hasLock=true; GCMultipleMergedRegionsProcedure child=723d42c26e868b0fdc6394437270af3b, parents:[723d42c26e868b0fdc6394437270af3b], [13c2b3011fe815032da3b2ae64bd9ecb]:java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hbase.client.Result.rawCells()" because the return value of "org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(org.apache.hadoop.hbase.client.RegionInfo)" is null; GCMultipleMergedRegionsProcedure child=723d42c26e868b0fdc6394437270af3b, parents:[723d42c26e868b0fdc6394437270af3b], [13c2b3011fe815032da3b2ae64bd9ecb] exec-time=5 msec 2024-11-10T05:13:36,297 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-10T05:13:36,299 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testCleanMerge Thread=29 (was 24) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testCleanMerge/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215616197 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PEWorker-1 app//org.apache.hadoop.hbase.master.assignment.RegionStateStore.getRegionCatalogResult(RegionStateStore.java:345) app//org.apache.hadoop.hbase.master.assignment.RegionStateStore.getMergeRegions(RegionStateStore.java:474) app//org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:103) app//org.apache.hadoop.hbase.master.assignment.GCMultipleMergedRegionsProcedure.executeFromState(GCMultipleMergedRegionsProcedure.java:45) app//org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) app//org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread$$Lambda$329/0x00007fa10c560430.call(Unknown Source) app//org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) - Thread LEAK? -, OpenFileDescriptor=355 (was 321) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1024 (was 1024), ProcessCount=11 (was 11), AvailableMemoryMB=3045 (was 3053) 2024-11-10T05:13:36,308 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testScanDoesNotCleanRegionsWithExistingParents Thread=28, OpenFileDescriptor=355, MaxFileDescriptor=1048576, SystemLoadAverage=1024, ProcessCount=11, AvailableMemoryMB=3046 2024-11-10T05:13:36,321 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents with version=8 2024-11-10T05:13:36,321 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/hbase.id] with ID: 1934d4b6-f412-4239-8003-c1aa8d2ec044 2024-11-10T05:13:36,322 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/.tmp/hbase.id 2024-11-10T05:13:36,328 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/hbase.id] 2024-11-10T05:13:36,363 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:13:36,363 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:13:36,367 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:13:36,368 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:13:36,392 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/MasterData/data/master/store 2024-11-10T05:13:36,403 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:36,403 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:13:36,403 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:36,403 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:36,404 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:13:36,404 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:36,404 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:36,404 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731215616403Disabling compacts and flushes for region at 1731215616403Disabling writes for close at 1731215616404 (+1 ms)Writing region close event to WAL at 1731215616404Closed at 1731215616404 2024-11-10T05:13:36,404 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/MasterData/data/master/store/.initializing 2024-11-10T05:13:36,405 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T05:13:36,412 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:13:36,417 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/MasterData/oldWALs, maxLogs=10 2024-11-10T05:13:36,442 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215616418 2024-11-10T05:13:36,443 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:13:36,443 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:13:36,443 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:36,444 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,444 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,446 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,453 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:13:36,453 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:13:36,454 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:13:36,459 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:36,460 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,464 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:13:36,464 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,464 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:36,465 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,468 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:13:36,469 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:36,469 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:36,470 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,471 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,472 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,472 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,472 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:13:36,473 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:36,483 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:13:36,484 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61021652, jitterRate=-0.09070652723312378}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:13:36,484 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731215616445Initializing all the Stores at 1731215616445Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731215616445Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215616445Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215616446 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215616446Cleaning up temporary data from old regions at 1731215616472 (+26 ms)Region opened successfully at 1731215616484 (+12 ms) 2024-11-10T05:13:36,485 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:13:36,492 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:13:36,493 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T05:13:36,493 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T05:13:36,495 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731215646495 2024-11-10T05:13:36,495 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:13:36,496 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T05:13:36,497 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T05:13:36,497 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T05:13:36,497 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T05:13:36,497 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T05:13:36,497 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T05:13:36,497 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T05:13:36,497 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T05:13:36,498 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T05:13:36,498 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T05:13:39,530 INFO [Time-limited test {}] janitor.TestCatalogJanitor(451): parent=0163d7ad86c87f443a08d865cc98936e, splita=7bdec5428ca4d5acf5b1a607128db534 2024-11-10T05:13:39,530 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:39,569 INFO [Time-limited test {}] janitor.TestCatalogJanitor(454): Created reference file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/data/default/testScanDoesNotCleanRegionsWithExistingParents/7bdec5428ca4d5acf5b1a607128db534/cf/1731215619530.0163d7ad86c87f443a08d865cc98936e 2024-11-10T05:13:39,569 DEBUG [Time-limited test {}] janitor.CatalogJanitor(180): 2024-11-10T05:13:39,570 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 0163d7ad86c87f443a08d865cc98936e, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731215616498.0163d7ad86c87f443a08d865cc98936e.', STARTKEY => 'aaa', ENDKEY => '', SPLIT => true} 2024-11-10T05:13:39,572 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:39,574 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => 0163d7ad86c87f443a08d865cc98936e, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731215616498.0163d7ad86c87f443a08d865cc98936e.', STARTKEY => 'aaa', ENDKEY => '', SPLIT => true} because daughter {ENCODED => 7bdec5428ca4d5acf5b1a607128db534, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731215617499.7bdec5428ca4d5acf5b1a607128db534.', STARTKEY => 'aaa', ENDKEY => 'ccc', SPLIT => true} still has references 2024-11-10T05:13:39,575 DEBUG [Time-limited test {}] janitor.CatalogJanitor(180): 2024-11-10T05:13:39,576 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 0163d7ad86c87f443a08d865cc98936e, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731215616498.0163d7ad86c87f443a08d865cc98936e.', STARTKEY => 'aaa', ENDKEY => '', SPLIT => true} 2024-11-10T05:13:39,577 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:39,577 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 0163d7ad86c87f443a08d865cc98936e because daughters -- 7bdec5428ca4d5acf5b1a607128db534, 26b1eea52e8aa56b9c72b03532ca9670 -- no longer hold references 2024-11-10T05:13:39,577 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=0163d7ad86c87f443a08d865cc98936e 2024-11-10T05:13:39,578 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=0163d7ad86c87f443a08d865cc98936e for split parent {ENCODED => 0163d7ad86c87f443a08d865cc98936e, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731215616498.0163d7ad86c87f443a08d865cc98936e.', STARTKEY => 'aaa', ENDKEY => '', SPLIT => true} 2024-11-10T05:13:39,578 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 7bdec5428ca4d5acf5b1a607128db534, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731215617499.7bdec5428ca4d5acf5b1a607128db534.', STARTKEY => 'aaa', ENDKEY => 'ccc', OFFLINE => true, SPLIT => true} 2024-11-10T05:13:39,578 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 7bdec5428ca4d5acf5b1a607128db534 because daughters -- 1f12729dd83b16b58e3dc297df3653a3, 07a8e48f918ec32ad53ffd28437e1462 -- no longer hold references 2024-11-10T05:13:39,578 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=7bdec5428ca4d5acf5b1a607128db534 2024-11-10T05:13:39,578 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=0163d7ad86c87f443a08d865cc98936e 2024-11-10T05:13:39,579 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=7bdec5428ca4d5acf5b1a607128db534 for split parent {ENCODED => 7bdec5428ca4d5acf5b1a607128db534, NAME => 'testScanDoesNotCleanRegionsWithExistingParents,aaa,1731215617499.7bdec5428ca4d5acf5b1a607128db534.', STARTKEY => 'aaa', ENDKEY => 'ccc', OFFLINE => true, SPLIT => true} 2024-11-10T05:13:39,579 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:13:39,579 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testScanDoesNotCleanRegionsWithExistingParents, region=7bdec5428ca4d5acf5b1a607128db534 2024-11-10T05:13:39,579 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:13:39,580 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:13:39,580 DEBUG [PEWorker-2 {}] assignment.GCRegionProcedure(87): Archiving region=7bdec5428ca4d5acf5b1a607128db534 2024-11-10T05:13:39,581 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/data/default/testScanDoesNotCleanRegionsWithExistingParents/7bdec5428ca4d5acf5b1a607128db534 2024-11-10T05:13:39,582 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/data/default/testScanDoesNotCleanRegionsWithExistingParents/7bdec5428ca4d5acf5b1a607128db534/cf] 2024-11-10T05:13:39,587 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testScanDoesNotCleanRegionsWithExistingParents Thread=33 (was 28) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215616418 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PEWorker-2 java.base@17.0.11/java.lang.String.trim(String.java:3534) app//org.apache.hadoop.conf.Configuration.handleDeprecation(Configuration.java:727) app//org.apache.hadoop.conf.Configuration.get(Configuration.java:1264) app//org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:1321) app//org.apache.hadoop.conf.Configuration.getTrimmed(Configuration.java:1341) app//org.apache.hadoop.fs.FileSystem.getDefaultUri(FileSystem.java:299) app//org.apache.hadoop.fs.FileSystem.get(FileSystem.java:545) app//org.apache.hadoop.fs.Path.getFileSystem(Path.java:366) app//org.apache.hadoop.hbase.util.CommonFSUtils.getRootDir(CommonFSUtils.java:292) app//org.apache.hadoop.hbase.backup.HFileArchiver.archiveRegion(HFileArchiver.java:99) app//org.apache.hadoop.hbase.master.assignment.GCRegionProcedure.executeFromState(GCRegionProcedure.java:89) app//org.apache.hadoop.hbase.master.assignment.GCRegionProcedure.executeFromState(GCRegionProcedure.java:51) app//org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) app//org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread$$Lambda$329/0x00007fa10c560430.call(Unknown Source) app//org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=389 (was 355) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1024 (was 1024), ProcessCount=11 (was 11), AvailableMemoryMB=2945 (was 3046) 2024-11-10T05:13:39,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testDuplicateHFileResolution Thread=34, OpenFileDescriptor=397, MaxFileDescriptor=1048576, SystemLoadAverage=1024, ProcessCount=12, AvailableMemoryMB=2943 2024-11-10T05:13:39,606 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testScanDoesNotCleanRegionsWithExistingParents/data/default/testScanDoesNotCleanRegionsWithExistingParents/7bdec5428ca4d5acf5b1a607128db534 2024-11-10T05:13:39,617 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution with version=8 2024-11-10T05:13:39,617 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/hbase.id] with ID: 084851b0-a77d-4ec4-a81d-0bd249a4c22a 2024-11-10T05:13:39,617 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/.tmp/hbase.id 2024-11-10T05:13:39,623 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/hbase.id] 2024-11-10T05:13:39,658 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:13:39,658 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:13:39,660 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:13:39,661 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:13:39,686 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/MasterData/data/master/store 2024-11-10T05:13:39,695 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:39,695 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:13:39,695 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:39,695 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:39,695 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:13:39,695 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:39,695 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:39,696 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731215619695Disabling compacts and flushes for region at 1731215619695Disabling writes for close at 1731215619695Writing region close event to WAL at 1731215619695Closed at 1731215619695 2024-11-10T05:13:39,696 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/MasterData/data/master/store/.initializing 2024-11-10T05:13:39,696 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T05:13:39,702 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:13:39,705 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/MasterData/oldWALs, maxLogs=10 2024-11-10T05:13:39,723 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215619706 2024-11-10T05:13:39,724 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:13:39,724 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:13:39,724 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:39,725 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,725 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,726 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,730 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:13:39,730 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:39,730 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:13:39,730 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,734 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:13:39,734 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:39,734 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:39,734 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,738 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:13:39,738 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:39,739 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:39,739 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:13:39,744 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:39,744 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:39,745 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,746 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,746 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,746 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,747 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,747 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:13:39,747 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:39,756 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:13:39,756 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61814749, jitterRate=-0.07888846099376678}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:13:39,756 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731215619725Initializing all the Stores at 1731215619725Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731215619725Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215619725Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215619725Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215619726 (+1 ms)Cleaning up temporary data from old regions at 1731215619747 (+21 ms)Region opened successfully at 1731215619756 (+9 ms) 2024-11-10T05:13:39,757 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:13:39,761 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:13:39,761 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T05:13:39,761 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T05:13:39,763 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731215649763 2024-11-10T05:13:39,763 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:13:39,765 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T05:13:39,765 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T05:13:39,766 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T05:13:39,766 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T05:13:39,766 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T05:13:39,766 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T05:13:39,766 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T05:13:39,766 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T05:13:39,766 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T05:13:39,767 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T05:13:39,768 INFO [Time-limited test {}] janitor.TestCatalogJanitor(639): Old root:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution 2024-11-10T05:13:39,768 INFO [Time-limited test {}] janitor.TestCatalogJanitor(640): Old table:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution 2024-11-10T05:13:39,768 INFO [Time-limited test {}] janitor.TestCatalogJanitor(641): Old store:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf 2024-11-10T05:13:39,768 INFO [Time-limited test {}] janitor.TestCatalogJanitor(645): Old archive:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf 2024-11-10T05:13:39,807 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(711): Adding 2 store files to the storedir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf 2024-11-10T05:13:39,809 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 3bbdd7e39f7aa4cc961d8d8f61528e9a, NAME => 'testDuplicateHFileResolution,aaa,1731215619767.3bbdd7e39f7aa4cc961d8d8f61528e9a.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:39,810 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:39,811 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 3bbdd7e39f7aa4cc961d8d8f61528e9a because daughters -- 3bbdd7e39f7aa4cc961d8d8f61528e9a, f130c70f5b385bc9f2dc9e662a052358 -- no longer hold references 2024-11-10T05:13:39,812 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=3bbdd7e39f7aa4cc961d8d8f61528e9a 2024-11-10T05:13:39,812 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=3bbdd7e39f7aa4cc961d8d8f61528e9a for split parent {ENCODED => 3bbdd7e39f7aa4cc961d8d8f61528e9a, NAME => 'testDuplicateHFileResolution,aaa,1731215619767.3bbdd7e39f7aa4cc961d8d8f61528e9a.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:39,812 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=3bbdd7e39f7aa4cc961d8d8f61528e9a 2024-11-10T05:13:39,813 DEBUG [PEWorker-1 {}] assignment.GCRegionProcedure(87): Archiving region=3bbdd7e39f7aa4cc961d8d8f61528e9a 2024-11-10T05:13:39,813 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a 2024-11-10T05:13:39,814 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf] 2024-11-10T05:13:39,831 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store0 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store0 2024-11-10T05:13:39,832 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store1 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store1 2024-11-10T05:13:39,832 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a 2024-11-10T05:13:39,833 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDuplicateHFileResolution,aaa,1731215619767.3bbdd7e39f7aa4cc961d8d8f61528e9a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731215619833"}]},"ts":"9223372036854775807"} 2024-11-10T05:13:39,834 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T05:13:39,834 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3bbdd7e39f7aa4cc961d8d8f61528e9a, NAME => 'testDuplicateHFileResolution,aaa,1731215619767.3bbdd7e39f7aa4cc961d8d8f61528e9a.', STARTKEY => 'aaa', ENDKEY => 'eee'}] 2024-11-10T05:13:39,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=3bbdd7e39f7aa4cc961d8d8f61528e9a in 23 msec 2024-11-10T05:13:40,088 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(711): Adding 2 store files to the storedir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf 2024-11-10T05:13:40,089 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 3bbdd7e39f7aa4cc961d8d8f61528e9a, NAME => 'testDuplicateHFileResolution,aaa,1731215619767.3bbdd7e39f7aa4cc961d8d8f61528e9a.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:40,091 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:40,093 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 3bbdd7e39f7aa4cc961d8d8f61528e9a because daughters -- 3bbdd7e39f7aa4cc961d8d8f61528e9a, f130c70f5b385bc9f2dc9e662a052358 -- no longer hold references 2024-11-10T05:13:40,093 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=3bbdd7e39f7aa4cc961d8d8f61528e9a 2024-11-10T05:13:40,093 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=3bbdd7e39f7aa4cc961d8d8f61528e9a for split parent {ENCODED => 3bbdd7e39f7aa4cc961d8d8f61528e9a, NAME => 'testDuplicateHFileResolution,aaa,1731215619767.3bbdd7e39f7aa4cc961d8d8f61528e9a.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:40,093 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=3bbdd7e39f7aa4cc961d8d8f61528e9a 2024-11-10T05:13:40,094 DEBUG [PEWorker-2 {}] assignment.GCRegionProcedure(87): Archiving region=3bbdd7e39f7aa4cc961d8d8f61528e9a 2024-11-10T05:13:40,094 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a 2024-11-10T05:13:40,094 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf] 2024-11-10T05:13:40,095 ERROR [PEWorker-2 {}] backup.HFileArchiver(523): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store0 already exists in archive, moving to timestamped backup and overwriting current FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store0. archiveLen: 12 currentLen: 12 archiveMtime: 1731215619830 currentMtime: 1731215620076 2024-11-10T05:13:40,096 INFO [PEWorker-2 {}] backup.HFileArchiver(540): Backed up archive file from file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store0 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store0.1731215620094. 2024-11-10T05:13:40,096 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store0 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store0 2024-11-10T05:13:40,097 ERROR [PEWorker-2 {}] backup.HFileArchiver(523): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store1 already exists in archive, moving to timestamped backup and overwriting current FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store1. archiveLen: 12 currentLen: 12 archiveMtime: 1731215619831 currentMtime: 1731215620080 2024-11-10T05:13:40,097 INFO [PEWorker-2 {}] backup.HFileArchiver(540): Backed up archive file from file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store1 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store1.1731215620094. 2024-11-10T05:13:40,097 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store1 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/archive/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a/cf/_store1 2024-11-10T05:13:40,098 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/data/default/testDuplicateHFileResolution/3bbdd7e39f7aa4cc961d8d8f61528e9a 2024-11-10T05:13:40,099 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testDuplicateHFileResolution,aaa,1731215619767.3bbdd7e39f7aa4cc961d8d8f61528e9a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731215620098"}]},"ts":"9223372036854775807"} 2024-11-10T05:13:40,099 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T05:13:40,099 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 3bbdd7e39f7aa4cc961d8d8f61528e9a, NAME => 'testDuplicateHFileResolution,aaa,1731215619767.3bbdd7e39f7aa4cc961d8d8f61528e9a.', STARTKEY => 'aaa', ENDKEY => 'eee'}] 2024-11-10T05:13:40,099 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, state=SUCCESS, hasLock=false; GCRegionProcedure table=testDuplicateHFileResolution, region=3bbdd7e39f7aa4cc961d8d8f61528e9a in 6 msec 2024-11-10T05:13:40,345 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:13:40,345 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:13:40,345 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:13:40,352 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testDuplicateHFileResolution Thread=37 (was 34) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215619706 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDuplicateHFileResolution/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=423 (was 397) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1024 (was 1024), ProcessCount=11 (was 12), AvailableMemoryMB=2931 (was 2943) 2024-11-10T05:13:40,358 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testAlreadyRunningStatus Thread=37, OpenFileDescriptor=423, MaxFileDescriptor=1048576, SystemLoadAverage=1024, ProcessCount=11, AvailableMemoryMB=2931 2024-11-10T05:13:40,369 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus with version=8 2024-11-10T05:13:40,369 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/hbase.id] with ID: 873b675d-51ed-4e1c-a3cb-7d56b39ac21c 2024-11-10T05:13:40,370 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/.tmp/hbase.id 2024-11-10T05:13:40,378 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/hbase.id] 2024-11-10T05:13:40,417 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:13:40,418 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:13:40,419 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:13:40,420 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:13:40,466 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/MasterData/data/master/store 2024-11-10T05:13:40,480 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:40,480 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:13:40,480 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:40,481 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:40,481 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:13:40,481 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:40,481 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:40,481 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731215620480Disabling compacts and flushes for region at 1731215620480Disabling writes for close at 1731215620481 (+1 ms)Writing region close event to WAL at 1731215620481Closed at 1731215620481 2024-11-10T05:13:40,481 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/MasterData/data/master/store/.initializing 2024-11-10T05:13:40,482 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T05:13:40,489 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:13:40,497 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/MasterData/oldWALs, maxLogs=10 2024-11-10T05:13:40,522 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215620497 2024-11-10T05:13:40,523 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:13:40,523 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:13:40,523 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:40,523 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,523 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,528 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:13:40,532 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:40,532 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:13:40,533 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,538 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:13:40,538 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:40,539 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:40,539 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,545 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:13:40,545 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:40,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:40,546 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:13:40,550 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:40,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:40,551 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,552 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,552 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,552 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,552 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,552 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:13:40,553 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,565 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:13:40,565 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71394195, jitterRate=0.06385640799999237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:13:40,565 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731215620524Initializing all the Stores at 1731215620524Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731215620524Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215620527 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215620527Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215620527Cleaning up temporary data from old regions at 1731215620552 (+25 ms)Region opened successfully at 1731215620565 (+13 ms) 2024-11-10T05:13:40,566 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:13:40,571 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:13:40,572 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T05:13:40,572 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T05:13:40,573 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731215650573 2024-11-10T05:13:40,573 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:13:40,574 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T05:13:40,574 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T05:13:40,574 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T05:13:40,574 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T05:13:40,575 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T05:13:40,575 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T05:13:40,575 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T05:13:40,575 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T05:13:40,575 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T05:13:40,575 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T05:13:40,577 DEBUG [Thread-266 {}] janitor.CatalogJanitor(170): CatalogJanitor already running 2024-11-10T05:13:40,579 DEBUG [Thread-265 {}] janitor.CatalogJanitor(180): 2024-11-10T05:13:40,580 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:13:40,580 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:13:40,580 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:13:40,586 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testAlreadyRunningStatus Thread=41 (was 37) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215620497 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testAlreadyRunningStatus/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=457 (was 423) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1024 (was 1024), ProcessCount=11 (was 11), AvailableMemoryMB=2917 (was 2931) 2024-11-10T05:13:40,592 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testParentCleanedEvenIfDaughterGoneFirst Thread=41, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=1024, ProcessCount=11, AvailableMemoryMB=2917 2024-11-10T05:13:40,606 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst with version=8 2024-11-10T05:13:40,606 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/hbase.id] with ID: 1de51430-b1bc-4076-8da2-5a6d52e9d5e2 2024-11-10T05:13:40,606 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/.tmp/hbase.id 2024-11-10T05:13:40,612 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/hbase.id] 2024-11-10T05:13:40,642 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:13:40,643 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:13:40,645 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:13:40,645 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:13:40,689 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store 2024-11-10T05:13:40,708 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:40,709 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:13:40,709 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:40,709 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:40,709 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:13:40,709 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:40,710 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:40,710 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731215620709Disabling compacts and flushes for region at 1731215620709Disabling writes for close at 1731215620709Writing region close event to WAL at 1731215620709Closed at 1731215620709 2024-11-10T05:13:40,710 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/.initializing 2024-11-10T05:13:40,710 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T05:13:40,719 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:13:40,729 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/MasterData/oldWALs, maxLogs=10 2024-11-10T05:13:40,760 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215620729 2024-11-10T05:13:40,761 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:13:40,762 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:13:40,762 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:40,762 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,763 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,768 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,777 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:13:40,777 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:40,778 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:13:40,778 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:13:40,783 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:40,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:40,784 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,791 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:13:40,792 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:40,792 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:40,792 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:13:40,801 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:40,801 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:40,802 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,803 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,803 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,804 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,804 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,804 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:13:40,805 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:40,836 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:13:40,836 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62914588, jitterRate=-0.06249958276748657}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:13:40,837 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731215620763Initializing all the Stores at 1731215620763Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731215620763Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215620768 (+5 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215620768Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215620768Cleaning up temporary data from old regions at 1731215620804 (+36 ms)Region opened successfully at 1731215620837 (+33 ms) 2024-11-10T05:13:40,840 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:13:40,857 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:13:40,857 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T05:13:40,857 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T05:13:40,876 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731215650876 2024-11-10T05:13:40,877 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:13:40,880 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T05:13:40,880 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T05:13:40,881 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T05:13:40,881 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T05:13:40,881 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T05:13:40,881 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T05:13:40,881 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T05:13:40,881 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T05:13:40,881 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T05:13:40,881 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T05:13:43,886 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:43,910 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => b056ef7160591931fea54c096d52c26b, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731215620882.b056ef7160591931fea54c096d52c26b.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:43,911 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:43,911 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => b056ef7160591931fea54c096d52c26b, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731215620882.b056ef7160591931fea54c096d52c26b.', STARTKEY => 'aaa', ENDKEY => 'eee'} because daughter {ENCODED => a92d420c8aaf93998e6e66c8d50a1598, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731215621883.a92d420c8aaf93998e6e66c8d50a1598.', STARTKEY => 'aaa', ENDKEY => 'ccc'} still has references 2024-11-10T05:13:43,912 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 1a756e39ce09cc3c44fe37770b96190b, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,ccc,1731215622884.1a756e39ce09cc3c44fe37770b96190b.', STARTKEY => 'ccc', ENDKEY => 'eee'} 2024-11-10T05:13:43,912 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 1a756e39ce09cc3c44fe37770b96190b because daughters -- 84ba3b4c32a926c4130a0a1d4b01f0a8, 09058425937b98433fbab095f9fa44b6 -- no longer hold references 2024-11-10T05:13:43,912 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=1a756e39ce09cc3c44fe37770b96190b 2024-11-10T05:13:43,913 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=1a756e39ce09cc3c44fe37770b96190b for split parent {ENCODED => 1a756e39ce09cc3c44fe37770b96190b, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,ccc,1731215622884.1a756e39ce09cc3c44fe37770b96190b.', STARTKEY => 'ccc', ENDKEY => 'eee'} 2024-11-10T05:13:43,913 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=1a756e39ce09cc3c44fe37770b96190b 2024-11-10T05:13:43,913 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:43,915 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testParentCleanedEvenIfDaughterGoneFirst,ccc,1731215622884.1a756e39ce09cc3c44fe37770b96190b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731215623914"}]},"ts":"9223372036854775807"} 2024-11-10T05:13:43,915 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T05:13:43,915 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1a756e39ce09cc3c44fe37770b96190b, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,ccc,1731215622884.1a756e39ce09cc3c44fe37770b96190b.', STARTKEY => 'ccc', ENDKEY => 'eee'}] 2024-11-10T05:13:43,916 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=1a756e39ce09cc3c44fe37770b96190b in 3 msec 2024-11-10T05:13:43,941 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:43,960 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => a92d420c8aaf93998e6e66c8d50a1598, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731215621883.a92d420c8aaf93998e6e66c8d50a1598.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T05:13:43,962 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:43,963 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:43,964 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => a92d420c8aaf93998e6e66c8d50a1598, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731215621883.a92d420c8aaf93998e6e66c8d50a1598.', STARTKEY => 'aaa', ENDKEY => 'ccc'} because daughter {ENCODED => 38f48f95843a072681f1a2cfb127372c, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731215622884.38f48f95843a072681f1a2cfb127372c.', STARTKEY => 'aaa', ENDKEY => 'bbb'} still has references 2024-11-10T05:13:43,964 DEBUG [Time-limited test {}] janitor.CatalogJanitor(368): Deferring removal of region {ENCODED => a92d420c8aaf93998e6e66c8d50a1598, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731215621883.a92d420c8aaf93998e6e66c8d50a1598.', STARTKEY => 'aaa', ENDKEY => 'ccc'} because daughter {ENCODED => 07e80fcfa941a85b84b41f5637c2635a, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,bbb,1731215622884.07e80fcfa941a85b84b41f5637c2635a.', STARTKEY => 'bbb', ENDKEY => 'ccc'} still has references 2024-11-10T05:13:43,965 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => a92d420c8aaf93998e6e66c8d50a1598, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731215621883.a92d420c8aaf93998e6e66c8d50a1598.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T05:13:43,966 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:43,967 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:43,967 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region a92d420c8aaf93998e6e66c8d50a1598 because daughters -- 38f48f95843a072681f1a2cfb127372c, 07e80fcfa941a85b84b41f5637c2635a -- no longer hold references 2024-11-10T05:13:43,967 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=a92d420c8aaf93998e6e66c8d50a1598 2024-11-10T05:13:43,968 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=a92d420c8aaf93998e6e66c8d50a1598 for split parent {ENCODED => a92d420c8aaf93998e6e66c8d50a1598, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731215621883.a92d420c8aaf93998e6e66c8d50a1598.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T05:13:43,968 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=a92d420c8aaf93998e6e66c8d50a1598 2024-11-10T05:13:43,968 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => b056ef7160591931fea54c096d52c26b, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731215620882.b056ef7160591931fea54c096d52c26b.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:43,968 DEBUG [PEWorker-2 {}] assignment.GCRegionProcedure(87): Archiving region=a92d420c8aaf93998e6e66c8d50a1598 2024-11-10T05:13:43,968 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/data/default/testParentCleanedEvenIfDaughterGoneFirst/a92d420c8aaf93998e6e66c8d50a1598 2024-11-10T05:13:43,969 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:43,969 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/data/default/testParentCleanedEvenIfDaughterGoneFirst/a92d420c8aaf93998e6e66c8d50a1598/cf] 2024-11-10T05:13:43,969 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region b056ef7160591931fea54c096d52c26b because daughters -- a92d420c8aaf93998e6e66c8d50a1598, 1a756e39ce09cc3c44fe37770b96190b -- no longer hold references 2024-11-10T05:13:43,969 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=b056ef7160591931fea54c096d52c26b 2024-11-10T05:13:43,970 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=b056ef7160591931fea54c096d52c26b for split parent {ENCODED => b056ef7160591931fea54c096d52c26b, NAME => 'testParentCleanedEvenIfDaughterGoneFirst,aaa,1731215620882.b056ef7160591931fea54c096d52c26b.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:43,970 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:13:43,970 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testParentCleanedEvenIfDaughterGoneFirst, region=b056ef7160591931fea54c096d52c26b 2024-11-10T05:13:43,970 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:13:43,970 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:13:43,992 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testParentCleanedEvenIfDaughterGoneFirst Thread=47 (was 41) Potentially hanging thread: PEWorker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2176) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215620729 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=499 (was 457) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1038 (was 1024) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2887 (was 2917) 2024-11-10T05:13:44,001 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testParentCleanedEvenIfDaughterGoneFirst/data/default/testParentCleanedEvenIfDaughterGoneFirst/a92d420c8aaf93998e6e66c8d50a1598 2024-11-10T05:13:44,008 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testLastParentCleanedEvenIfDaughterGoneFirst Thread=46, OpenFileDescriptor=501, MaxFileDescriptor=1048576, SystemLoadAverage=1038, ProcessCount=11, AvailableMemoryMB=2886 2024-11-10T05:13:44,025 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst with version=8 2024-11-10T05:13:44,026 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/hbase.id] with ID: f245bf48-1c53-49e4-8750-76f8f3ad901a 2024-11-10T05:13:44,026 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/.tmp/hbase.id 2024-11-10T05:13:44,035 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/hbase.id] 2024-11-10T05:13:44,081 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:13:44,081 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:13:44,083 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:13:44,083 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:13:44,165 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store 2024-11-10T05:13:44,176 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:44,176 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:13:44,176 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:44,176 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:44,176 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:13:44,176 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:44,177 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:44,177 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731215624176Disabling compacts and flushes for region at 1731215624176Disabling writes for close at 1731215624176Writing region close event to WAL at 1731215624176Closed at 1731215624176 2024-11-10T05:13:44,177 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/.initializing 2024-11-10T05:13:44,177 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T05:13:44,184 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:13:44,188 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/oldWALs, maxLogs=10 2024-11-10T05:13:44,215 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215624189 2024-11-10T05:13:44,216 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:13:44,216 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:13:44,216 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:44,216 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,217 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,218 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,221 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:13:44,221 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:44,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:13:44,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,241 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:13:44,241 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:44,242 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:44,242 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,255 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:13:44,255 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:44,255 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:44,256 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,260 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:13:44,261 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:44,261 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:44,261 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,262 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,262 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,263 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,263 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,263 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:13:44,264 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:44,276 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:13:44,276 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63740956, jitterRate=-0.05018573999404907}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:13:44,276 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731215624217Initializing all the Stores at 1731215624217Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731215624217Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215624218 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215624218Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215624218Cleaning up temporary data from old regions at 1731215624263 (+45 ms)Region opened successfully at 1731215624276 (+13 ms) 2024-11-10T05:13:44,280 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:13:44,295 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:13:44,295 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T05:13:44,296 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T05:13:44,312 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731215654312 2024-11-10T05:13:44,312 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:13:44,321 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T05:13:44,321 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T05:13:44,321 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T05:13:44,322 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T05:13:44,322 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T05:13:44,322 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T05:13:44,324 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T05:13:44,325 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T05:13:44,325 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T05:13:44,325 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T05:13:44,479 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-10T05:13:44,480 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-10T05:13:47,330 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,350 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 7ccc78b3489f355994ea47e2e87725d7, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731215624325.7ccc78b3489f355994ea47e2e87725d7.', STARTKEY => 'aaa', ENDKEY => ''} 2024-11-10T05:13:47,352 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,352 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => 7ccc78b3489f355994ea47e2e87725d7, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731215624325.7ccc78b3489f355994ea47e2e87725d7.', STARTKEY => 'aaa', ENDKEY => ''} because daughter {ENCODED => 52d3d545bdb29ed852b8ecdc34d668a8, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731215625327.52d3d545bdb29ed852b8ecdc34d668a8.', STARTKEY => 'aaa', ENDKEY => 'ccc'} still has references 2024-11-10T05:13:47,353 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => b7014bdbcc65ebbf251f4ccae937c6fd, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,ccc,1731215626328.b7014bdbcc65ebbf251f4ccae937c6fd.', STARTKEY => 'ccc', ENDKEY => ''} 2024-11-10T05:13:47,353 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region b7014bdbcc65ebbf251f4ccae937c6fd because daughters -- b04922db11d1eeb08082d212030ec7c9, fc1d7f870096af8c1a9496c01600f97d -- no longer hold references 2024-11-10T05:13:47,354 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=b7014bdbcc65ebbf251f4ccae937c6fd 2024-11-10T05:13:47,354 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=b7014bdbcc65ebbf251f4ccae937c6fd for split parent {ENCODED => b7014bdbcc65ebbf251f4ccae937c6fd, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,ccc,1731215626328.b7014bdbcc65ebbf251f4ccae937c6fd.', STARTKEY => 'ccc', ENDKEY => ''} 2024-11-10T05:13:47,354 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=b7014bdbcc65ebbf251f4ccae937c6fd 2024-11-10T05:13:47,355 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,355 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testLastParentCleanedEvenIfDaughterGoneFirst,ccc,1731215626328.b7014bdbcc65ebbf251f4ccae937c6fd.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731215627355"}]},"ts":"9223372036854775807"} 2024-11-10T05:13:47,356 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T05:13:47,356 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => b7014bdbcc65ebbf251f4ccae937c6fd, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,ccc,1731215626328.b7014bdbcc65ebbf251f4ccae937c6fd.', STARTKEY => 'ccc', ENDKEY => ''}] 2024-11-10T05:13:47,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=b7014bdbcc65ebbf251f4ccae937c6fd in 3 msec 2024-11-10T05:13:47,369 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,382 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 52d3d545bdb29ed852b8ecdc34d668a8, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731215625327.52d3d545bdb29ed852b8ecdc34d668a8.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T05:13:47,383 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,384 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,385 DEBUG [Time-limited test {}] janitor.CatalogJanitor(364): Deferring removal of region {ENCODED => 52d3d545bdb29ed852b8ecdc34d668a8, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731215625327.52d3d545bdb29ed852b8ecdc34d668a8.', STARTKEY => 'aaa', ENDKEY => 'ccc'} because daughter {ENCODED => 225c45bbfe2f543c51a84321c17a26e8, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731215626328.225c45bbfe2f543c51a84321c17a26e8.', STARTKEY => 'aaa', ENDKEY => 'bbb'} still has references 2024-11-10T05:13:47,385 DEBUG [Time-limited test {}] janitor.CatalogJanitor(368): Deferring removal of region {ENCODED => 52d3d545bdb29ed852b8ecdc34d668a8, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731215625327.52d3d545bdb29ed852b8ecdc34d668a8.', STARTKEY => 'aaa', ENDKEY => 'ccc'} because daughter {ENCODED => 13375abdc0d9179648ddddb90d8fc5cb, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,bbb,1731215626328.13375abdc0d9179648ddddb90d8fc5cb.', STARTKEY => 'bbb', ENDKEY => 'ccc'} still has references 2024-11-10T05:13:47,385 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 52d3d545bdb29ed852b8ecdc34d668a8, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731215625327.52d3d545bdb29ed852b8ecdc34d668a8.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T05:13:47,386 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,387 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,387 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 52d3d545bdb29ed852b8ecdc34d668a8 because daughters -- 225c45bbfe2f543c51a84321c17a26e8, 13375abdc0d9179648ddddb90d8fc5cb -- no longer hold references 2024-11-10T05:13:47,387 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=52d3d545bdb29ed852b8ecdc34d668a8 2024-11-10T05:13:47,387 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=52d3d545bdb29ed852b8ecdc34d668a8 for split parent {ENCODED => 52d3d545bdb29ed852b8ecdc34d668a8, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731215625327.52d3d545bdb29ed852b8ecdc34d668a8.', STARTKEY => 'aaa', ENDKEY => 'ccc'} 2024-11-10T05:13:47,387 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 7ccc78b3489f355994ea47e2e87725d7, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731215624325.7ccc78b3489f355994ea47e2e87725d7.', STARTKEY => 'aaa', ENDKEY => ''} 2024-11-10T05:13:47,388 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=52d3d545bdb29ed852b8ecdc34d668a8 2024-11-10T05:13:47,389 DEBUG [PEWorker-2 {}] assignment.GCRegionProcedure(87): Archiving region=52d3d545bdb29ed852b8ecdc34d668a8 2024-11-10T05:13:47,389 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,389 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/data/default/testLastParentCleanedEvenIfDaughterGoneFirst/52d3d545bdb29ed852b8ecdc34d668a8 2024-11-10T05:13:47,389 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 7ccc78b3489f355994ea47e2e87725d7 because daughters -- 52d3d545bdb29ed852b8ecdc34d668a8, b7014bdbcc65ebbf251f4ccae937c6fd -- no longer hold references 2024-11-10T05:13:47,389 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=7ccc78b3489f355994ea47e2e87725d7 2024-11-10T05:13:47,389 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=7ccc78b3489f355994ea47e2e87725d7 for split parent {ENCODED => 7ccc78b3489f355994ea47e2e87725d7, NAME => 'testLastParentCleanedEvenIfDaughterGoneFirst,aaa,1731215624325.7ccc78b3489f355994ea47e2e87725d7.', STARTKEY => 'aaa', ENDKEY => ''} 2024-11-10T05:13:47,389 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:13:47,389 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/data/default/testLastParentCleanedEvenIfDaughterGoneFirst/52d3d545bdb29ed852b8ecdc34d668a8/cf] 2024-11-10T05:13:47,390 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:13:47,390 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=3, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testLastParentCleanedEvenIfDaughterGoneFirst, region=7ccc78b3489f355994ea47e2e87725d7 2024-11-10T05:13:47,390 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:13:47,403 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/data/default/testLastParentCleanedEvenIfDaughterGoneFirst/52d3d545bdb29ed852b8ecdc34d668a8 2024-11-10T05:13:47,406 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testLastParentCleanedEvenIfDaughterGoneFirst Thread=50 (was 46) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testLastParentCleanedEvenIfDaughterGoneFirst/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215624189 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=528 (was 501) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=987 (was 1038), ProcessCount=11 (was 11), AvailableMemoryMB=2852 (was 2886) 2024-11-10T05:13:47,412 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testSplitParentFirstComparator Thread=49, OpenFileDescriptor=525, MaxFileDescriptor=1048576, SystemLoadAverage=987, ProcessCount=11, AvailableMemoryMB=2851 2024-11-10T05:13:47,428 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator with version=8 2024-11-10T05:13:47,428 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/hbase.id] with ID: 7a1a35ee-5d2f-4b73-8ab8-ee49e2f5ef8c 2024-11-10T05:13:47,428 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/.tmp/hbase.id 2024-11-10T05:13:47,434 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/hbase.id] 2024-11-10T05:13:47,467 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:13:47,468 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:13:47,469 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:13:47,469 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:13:47,494 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/MasterData/data/master/store 2024-11-10T05:13:47,506 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:47,507 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:13:47,507 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:47,507 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:47,507 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:13:47,507 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:47,507 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:47,507 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731215627507Disabling compacts and flushes for region at 1731215627507Disabling writes for close at 1731215627507Writing region close event to WAL at 1731215627507Closed at 1731215627507 2024-11-10T05:13:47,508 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/MasterData/data/master/store/.initializing 2024-11-10T05:13:47,508 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T05:13:47,514 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:13:47,521 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/MasterData/oldWALs, maxLogs=10 2024-11-10T05:13:47,548 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215627521 2024-11-10T05:13:47,552 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:13:47,553 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:13:47,553 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:47,554 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,554 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,555 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:13:47,560 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:13:47,561 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:13:47,565 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:47,565 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,570 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:13:47,570 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,570 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:47,570 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,574 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:13:47,574 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,575 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:47,575 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,576 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,576 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,576 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,576 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,577 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:13:47,577 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,587 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:13:47,587 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74048949, jitterRate=0.10341532528400421}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:13:47,588 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731215627554Initializing all the Stores at 1731215627555 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731215627555Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215627555Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215627555Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215627555Cleaning up temporary data from old regions at 1731215627576 (+21 ms)Region opened successfully at 1731215627588 (+12 ms) 2024-11-10T05:13:47,588 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:13:47,593 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:13:47,594 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T05:13:47,594 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T05:13:47,599 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731215657598 2024-11-10T05:13:47,599 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:13:47,600 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T05:13:47,601 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T05:13:47,601 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T05:13:47,601 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T05:13:47,601 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T05:13:47,601 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T05:13:47,601 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T05:13:47,601 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T05:13:47,601 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T05:13:47,601 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T05:13:47,602 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:13:47,602 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:13:47,602 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:13:47,609 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testSplitParentFirstComparator Thread=53 (was 49) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215627521 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testSplitParentFirstComparator/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=559 (was 525) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=987 (was 987), ProcessCount=11 (was 11), AvailableMemoryMB=2841 (was 2851) 2024-11-10T05:13:47,616 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testDontCleanMergeIfFileSystemException Thread=53, OpenFileDescriptor=559, MaxFileDescriptor=1048576, SystemLoadAverage=987, ProcessCount=11, AvailableMemoryMB=2840 2024-11-10T05:13:47,629 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException with version=8 2024-11-10T05:13:47,630 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/hbase.id] with ID: bb0d85ea-6c53-4b26-9d2e-1c96c0607206 2024-11-10T05:13:47,630 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/.tmp/hbase.id 2024-11-10T05:13:47,638 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/hbase.id] 2024-11-10T05:13:47,701 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:13:47,701 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:13:47,703 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:13:47,703 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:13:47,741 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/MasterData/data/master/store 2024-11-10T05:13:47,772 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:47,772 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:13:47,772 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:47,772 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:47,772 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-10T05:13:47,772 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:47,772 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:47,773 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731215627772Disabling compacts and flushes for region at 1731215627772Disabling writes for close at 1731215627772Writing region close event to WAL at 1731215627772Closed at 1731215627772 2024-11-10T05:13:47,773 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/MasterData/data/master/store/.initializing 2024-11-10T05:13:47,773 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T05:13:47,787 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:13:47,792 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/MasterData/oldWALs, maxLogs=10 2024-11-10T05:13:47,815 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215627792 2024-11-10T05:13:47,815 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:13:47,815 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:13:47,815 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:47,816 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,816 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,817 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,825 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:13:47,825 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:13:47,826 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,831 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:13:47,831 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,832 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:47,832 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:13:47,841 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,841 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:47,842 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,847 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:13:47,848 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:47,848 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:47,848 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,849 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,849 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,852 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,852 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,853 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:13:47,854 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:47,873 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:13:47,873 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60950337, jitterRate=-0.09176920354366302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:13:47,873 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731215627816Initializing all the Stores at 1731215627816Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731215627816Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215627817 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215627817Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215627817Cleaning up temporary data from old regions at 1731215627852 (+35 ms)Region opened successfully at 1731215627873 (+21 ms) 2024-11-10T05:13:47,875 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:13:47,881 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:13:47,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T05:13:47,882 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T05:13:47,883 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731215657883 2024-11-10T05:13:47,883 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:13:47,884 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T05:13:47,884 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T05:13:47,884 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T05:13:47,885 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T05:13:47,885 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T05:13:47,885 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T05:13:47,885 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T05:13:47,885 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T05:13:47,885 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T05:13:47,885 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T05:13:47,886 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:48,320 DEBUG [Time-limited test {}] janitor.CatalogJanitor(258): Cleaning merged region {ENCODED => 3d14aaa1085d11eef7ef2f53526f981e, NAME => 'testDontCleanMergeIfFileSystemException,aaa,1731215627885.3d14aaa1085d11eef7ef2f53526f981e.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:48,324 ERROR [Time-limited test {}] janitor.CatalogJanitor(420): Error trying to determine if region exists, assuming exists and has references java.io.IOException: Some exception at org.apache.hadoop.hbase.util.CommonFSUtils.isExists(CommonFSUtils.java:673) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.janitor.CatalogJanitor.checkRegionReferences(CatalogJanitor.java:416) ~[classes/:?] at org.apache.hadoop.hbase.master.janitor.CatalogJanitor.cleanMergeRegion(CatalogJanitor.java:262) ~[classes/:?] at org.apache.hadoop.hbase.master.janitor.TestCatalogJanitor.testDontCleanMergeIfFileSystemException(TestCatalogJanitor.java:201) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T05:13:48,326 DEBUG [Time-limited test {}] janitor.CatalogJanitor(283): Deferring cleanup up of 2 parents of merged region 3d14aaa1085d11eef7ef2f53526f981e, because references still exist in merged region or we encountered an exception in checking 2024-11-10T05:13:48,327 DEBUG [Time-limited test {}] janitor.CatalogJanitor(258): Cleaning merged region {ENCODED => 3d14aaa1085d11eef7ef2f53526f981e, NAME => 'testDontCleanMergeIfFileSystemException,aaa,1731215627885.3d14aaa1085d11eef7ef2f53526f981e.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:48,329 ERROR [Time-limited test {}] janitor.CatalogJanitor(441): Error trying to determine if region 3d14aaa1085d11eef7ef2f53526f981e has references, assuming it does java.io.IOException: Some exception at org.apache.hadoop.hbase.master.janitor.TestCatalogJanitor.lambda$testDontCleanMergeIfFileSystemException$0(TestCatalogJanitor.java:211) ~[test-classes/:?] at org.mockito.internal.stubbing.StubbedInvocationMatcher.answer(StubbedInvocationMatcher.java:42) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.handler.MockHandlerImpl.handle(MockHandlerImpl.java:103) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.handler.NullResultGuardian.handle(NullResultGuardian.java:29) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.handler.InvocationNotifierHandler.handle(InvocationNotifierHandler.java:34) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:82) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor.doIntercept(MockMethodInterceptor.java:56) ~[mockito-core-4.11.0.jar:?] at org.mockito.internal.creation.bytebuddy.MockMethodInterceptor$DispatcherDefaultingToRealMethod.interceptSuperCallable(MockMethodInterceptor.java:145) ~[mockito-core-4.11.0.jar:?] at org.apache.hadoop.fs.LocalFileSystem$MockitoMock$wTCnPTMQ.exists(Unknown Source) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.regionserver.HRegionFileSystem.openRegionFromFileSystem(HRegionFileSystem.java:959) ~[classes/:?] at org.apache.hadoop.hbase.master.janitor.CatalogJanitor.checkRegionReferences(CatalogJanitor.java:428) ~[classes/:?] at org.apache.hadoop.hbase.master.janitor.CatalogJanitor.cleanMergeRegion(CatalogJanitor.java:262) ~[classes/:?] at org.apache.hadoop.hbase.master.janitor.TestCatalogJanitor.testDontCleanMergeIfFileSystemException(TestCatalogJanitor.java:214) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-10T05:13:48,332 DEBUG [Time-limited test {}] janitor.CatalogJanitor(283): Deferring cleanup up of 2 parents of merged region 3d14aaa1085d11eef7ef2f53526f981e, because references still exist in merged region or we encountered an exception in checking 2024-11-10T05:13:48,332 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:13:48,333 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:13:48,333 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:13:48,353 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testDontCleanMergeIfFileSystemException Thread=58 (was 53) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testDontCleanMergeIfFileSystemException/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215627792 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcExecTimeout java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:274) app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) - Thread LEAK? -, OpenFileDescriptor=593 (was 559) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=987 (was 987), ProcessCount=11 (was 11), AvailableMemoryMB=2775 (was 2840) 2024-11-10T05:13:48,367 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: master.janitor.TestCatalogJanitor#testArchiveOldRegion Thread=57, OpenFileDescriptor=595, MaxFileDescriptor=1048576, SystemLoadAverage=987, ProcessCount=11, AvailableMemoryMB=2774 2024-11-10T05:13:48,416 INFO [Time-limited test {}] util.FSUtils(489): Created version file at file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion with version=8 2024-11-10T05:13:48,418 DEBUG [Time-limited test {}] util.FSUtils(620): Create cluster ID file [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/hbase.id] with ID: 07d046ff-85ec-458b-81b2-d1742d714996 2024-11-10T05:13:48,418 DEBUG [Time-limited test {}] util.FSUtils(625): Write the cluster ID file to a temporary location: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/.tmp/hbase.id 2024-11-10T05:13:48,448 DEBUG [Time-limited test {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/.tmp/hbase.id]:[file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/hbase.id] 2024-11-10T05:13:48,624 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-10T05:13:48,625 INFO [Time-limited test {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-10T05:13:48,626 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-10T05:13:48,628 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-10T05:13:48,705 INFO [Time-limited test {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/MasterData/data/master/store 2024-11-10T05:13:48,715 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:48,715 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-10T05:13:48,715 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:48,715 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:48,716 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 1 ms 2024-11-10T05:13:48,716 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:48,716 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-10T05:13:48,716 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731215628715Disabling compacts and flushes for region at 1731215628715Disabling writes for close at 1731215628716 (+1 ms)Writing region close event to WAL at 1731215628716Closed at 1731215628716 2024-11-10T05:13:48,716 WARN [Time-limited test {}] region.MasterRegion(249): failed to clean up initializing flag: file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/MasterData/data/master/store/.initializing 2024-11-10T05:13:48,716 DEBUG [Time-limited test {}] region.MasterRegion(219): WALDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/MasterData/WALs/mockmaster.example.org,1234,-1 2024-11-10T05:13:48,722 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-10T05:13:48,729 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=mockmaster.example.org%2C1234%2C-1, suffix=, logDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/MasterData/WALs/mockmaster.example.org,1234,-1, archiveDir=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/MasterData/oldWALs, maxLogs=10 2024-11-10T05:13:48,780 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215628730 2024-11-10T05:13:48,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [] 2024-11-10T05:13:48,781 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-10T05:13:48,781 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-10T05:13:48,782 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,782 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,783 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,796 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-10T05:13:48,796 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:48,796 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-10T05:13:48,797 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-10T05:13:48,804 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:48,804 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:48,805 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-10T05:13:48,809 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:48,809 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:48,810 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-10T05:13:48,820 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:48,820 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-10T05:13:48,820 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,821 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,821 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,822 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,822 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,822 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-10T05:13:48,823 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-10T05:13:48,832 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-10T05:13:48,832 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67389846, jitterRate=0.004186958074569702}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-10T05:13:48,833 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731215628782Initializing all the Stores at 1731215628782Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731215628782Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215628783 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215628783Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731215628783Cleaning up temporary data from old regions at 1731215628822 (+39 ms)Region opened successfully at 1731215628832 (+10 ms) 2024-11-10T05:13:48,833 INFO [Time-limited test {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-10T05:13:48,839 INFO [Time-limited test {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-10T05:13:48,839 INFO [Time-limited test {}] procedure2.ProcedureExecutor(676): Recovered NoopProcedureStore lease in 0 msec 2024-11-10T05:13:48,840 INFO [Time-limited test {}] procedure2.ProcedureExecutor(690): Loaded NoopProcedureStore in 0 msec 2024-11-10T05:13:48,840 INFO [Time-limited test {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731215658840 2024-11-10T05:13:48,840 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-10T05:13:48,841 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,100,1 2024-11-10T05:13:48,841 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,101,1 2024-11-10T05:13:48,841 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,102,1 2024-11-10T05:13:48,841 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,103,1 2024-11-10T05:13:48,842 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,104,1 2024-11-10T05:13:48,842 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,105,1 2024-11-10T05:13:48,842 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,106,1 2024-11-10T05:13:48,842 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,107,1 2024-11-10T05:13:48,842 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,108,1 2024-11-10T05:13:48,842 INFO [Time-limited test {}] master.ServerManager(517): Registering regionserver=localhost,109,1 2024-11-10T05:13:48,843 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(566): Table dir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion 2024-11-10T05:13:48,843 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(567): Store dir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf 2024-11-10T05:13:48,843 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(568): Store archive dir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/archive/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf 2024-11-10T05:13:48,870 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(711): Adding 2 store files to the storedir:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf 2024-11-10T05:13:48,871 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(576): Have store file:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf/_store0 2024-11-10T05:13:48,871 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(576): Have store file:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf/_store1 2024-11-10T05:13:48,871 DEBUG [Time-limited test {}] janitor.CatalogJanitor(329): Cleaning parent region {ENCODED => 03378d56d892099c2417dda794e042e9, NAME => 'testArchiveOldRegion,aaa,1731215628842.03378d56d892099c2417dda794e042e9.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:48,872 DEBUG [Time-limited test {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-10T05:13:48,872 DEBUG [Time-limited test {}] janitor.CatalogJanitor(351): Deleting region 03378d56d892099c2417dda794e042e9 because daughters -- 03378d56d892099c2417dda794e042e9, cf6ffd9f1da2c6beba7bced3f271b44b -- no longer hold references 2024-11-10T05:13:48,872 DEBUG [Time-limited test {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testArchiveOldRegion, region=03378d56d892099c2417dda794e042e9 2024-11-10T05:13:48,873 DEBUG [Time-limited test {}] janitor.CatalogJanitor(358): Submitted procedure pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testArchiveOldRegion, region=03378d56d892099c2417dda794e042e9 for split parent {ENCODED => 03378d56d892099c2417dda794e042e9, NAME => 'testArchiveOldRegion,aaa,1731215628842.03378d56d892099c2417dda794e042e9.', STARTKEY => 'aaa', ENDKEY => 'eee'} 2024-11-10T05:13:48,873 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=1, state=RUNNABLE:GC_REGION_PREPARE, hasLock=false; GCRegionProcedure table=testArchiveOldRegion, region=03378d56d892099c2417dda794e042e9 2024-11-10T05:13:48,873 DEBUG [PEWorker-1 {}] assignment.GCRegionProcedure(87): Archiving region=03378d56d892099c2417dda794e042e9 2024-11-10T05:13:48,873 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9 2024-11-10T05:13:48,874 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf] 2024-11-10T05:13:48,897 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf/_store0 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/archive/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf/_store0 2024-11-10T05:13:48,898 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf/_store1 to file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/archive/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf/_store1 2024-11-10T05:13:48,898 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9 2024-11-10T05:13:48,899 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"testArchiveOldRegion,aaa,1731215628842.03378d56d892099c2417dda794e042e9.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731215628898"}]},"ts":"9223372036854775807"} 2024-11-10T05:13:48,899 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-10T05:13:48,899 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 03378d56d892099c2417dda794e042e9, NAME => 'testArchiveOldRegion,aaa,1731215628842.03378d56d892099c2417dda794e042e9.', STARTKEY => 'aaa', ENDKEY => 'eee'}] 2024-11-10T05:13:48,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; GCRegionProcedure table=testArchiveOldRegion, region=03378d56d892099c2417dda794e042e9 in 27 msec 2024-11-10T05:13:49,123 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(588): Finished cleanup of parent region 2024-11-10T05:13:49,124 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(606): Current archived files: 2024-11-10T05:13:49,124 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(608): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf/_store0 2024-11-10T05:13:49,124 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(608): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf/_store1 2024-11-10T05:13:49,124 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(606): Current archived files: 2024-11-10T05:13:49,124 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(608): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/archive/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf/_store0 2024-11-10T05:13:49,124 DEBUG [Time-limited test {}] janitor.TestCatalogJanitor(608): file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/archive/data/default/testArchiveOldRegion/03378d56d892099c2417dda794e042e9/cf/_store1 2024-11-10T05:13:49,129 INFO [Time-limited test {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-10T05:13:49,129 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-10T05:13:49,130 INFO [Time-limited test {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-10T05:13:49,138 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: master.janitor.TestCatalogJanitor#testArchiveOldRegion Thread=61 (was 57) Potentially hanging thread: AsyncFSOutputFlusher-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/MasterData/WALs/mockmaster.example.org,1234,-1/mockmaster.example.org%2C1234%2C-1.1731215628730 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/e6c43dce-46e0-ae59-2121-960a4743047f/testArchiveOldRegion/MasterData-prefix:mockmaster.example.org,1234,-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=627 (was 595) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=987 (was 987), ProcessCount=11 (was 11), AvailableMemoryMB=4181 (was 2774) - AvailableMemoryMB LEAK? -